diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" --- "a/wandb/debug-internal.log" +++ "b/wandb/debug-internal.log" @@ -1,1648 +1,93 @@ -2024-02-14 11:54:05,519 INFO StreamThr :19614 [internal.py:wandb_internal():86] W&B internal server running at pid: 19614, started at: 2024-02-14 11:54:05.518558 -2024-02-14 11:54:05,520 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status -2024-02-14 11:54:08,529 INFO WriterThread:19614 [datastore.py:open_for_write():87] open: /content/wandb/run-20240214_115405-518dheci/run-518dheci.wandb -2024-02-14 11:54:08,530 DEBUG SenderThread:19614 [sender.py:send():382] send: header -2024-02-14 11:54:08,532 DEBUG SenderThread:19614 [sender.py:send():382] send: run -2024-02-14 11:54:08,756 INFO SenderThread:19614 [dir_watcher.py:__init__():211] watching files in: /content/wandb/run-20240214_115405-518dheci/files -2024-02-14 11:54:08,756 INFO SenderThread:19614 [sender.py:_start_run_threads():1136] run started: 518dheci with start time 1707911645.519151 -2024-02-14 11:54:08,756 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: check_version -2024-02-14 11:54:08,757 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: check_version -2024-02-14 11:54:08,890 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: run_start -2024-02-14 11:54:08,899 DEBUG HandlerThread:19614 [system_info.py:__init__():27] System info init -2024-02-14 11:54:08,899 DEBUG HandlerThread:19614 [system_info.py:__init__():42] System info init done -2024-02-14 11:54:08,899 INFO HandlerThread:19614 [system_monitor.py:start():194] Starting system monitor -2024-02-14 11:54:08,899 INFO SystemMonitor:19614 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-02-14 11:54:08,900 INFO HandlerThread:19614 [system_monitor.py:probe():214] Collecting system info -2024-02-14 11:54:08,900 INFO SystemMonitor:19614 [interfaces.py:start():190] Started cpu monitoring -2024-02-14 11:54:08,901 INFO SystemMonitor:19614 [interfaces.py:start():190] Started disk monitoring -2024-02-14 11:54:08,902 INFO SystemMonitor:19614 [interfaces.py:start():190] Started gpu monitoring -2024-02-14 11:54:08,903 INFO SystemMonitor:19614 [interfaces.py:start():190] Started memory monitoring -2024-02-14 11:54:08,903 INFO SystemMonitor:19614 [interfaces.py:start():190] Started network monitoring -2024-02-14 11:54:08,913 DEBUG HandlerThread:19614 [system_info.py:probe():151] Probing system -2024-02-14 11:54:08,914 DEBUG HandlerThread:19614 [gitlib.py:_init_repo():56] git repository is invalid -2024-02-14 11:54:08,914 DEBUG HandlerThread:19614 [system_info.py:probe():199] Probing system done -2024-02-14 11:54:08,914 DEBUG HandlerThread:19614 [system_monitor.py:probe():223] {'os': 'Linux-6.1.58+-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-02-14T11:54:08.913392', 'startedAt': '2024-02-14T11:54:05.513049', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'Smart_Agriculture_Falcon.ipynb', 'codePathLocal': None, 'colab': 'https://colab.research.google.com/notebook#fileId=1kv2VO7q0c8xX_DQt5NERLeJJZtHW6zHO', 'host': 'f4b889916ade', 'username': 'root', 'executable': '/usr/bin/python3', 'cpu_count': 4, 'cpu_count_logical': 8, 'cpu_freq': {'current': 2299.998, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2299.998, 'min': 0.0, 'max': 0.0}, {'current': 2299.998, 'min': 0.0, 'max': 0.0}, {'current': 2299.998, 'min': 0.0, 'max': 0.0}, {'current': 2299.998, 'min': 0.0, 'max': 0.0}, {'current': 2299.998, 'min': 0.0, 'max': 0.0}, {'current': 2299.998, 'min': 0.0, 'max': 0.0}, {'current': 2299.998, 'min': 0.0, 'max': 0.0}, {'current': 2299.998, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 166.77415084838867, 'used': 52.78982162475586}}, 'gpu': 'Tesla T4', 'gpu_count': 1, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 50.993690490722656}} -2024-02-14 11:54:08,914 INFO HandlerThread:19614 [system_monitor.py:probe():224] Finished collecting system info -2024-02-14 11:54:08,914 INFO HandlerThread:19614 [system_monitor.py:probe():227] Publishing system info -2024-02-14 11:54:08,915 INFO HandlerThread:19614 [system_monitor.py:probe():229] Finished publishing system info -2024-02-14 11:54:08,919 DEBUG SenderThread:19614 [sender.py:send():382] send: files -2024-02-14 11:54:08,920 INFO SenderThread:19614 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now -2024-02-14 11:54:08,935 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: python_packages -2024-02-14 11:54:08,935 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: python_packages -2024-02-14 11:54:08,976 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 11:54:08,977 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 11:54:08,978 DEBUG SenderThread:19614 [sender.py:send():382] send: telemetry -2024-02-14 11:54:08,978 DEBUG SenderThread:19614 [sender.py:send():382] send: config -2024-02-14 11:54:08,978 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 11:54:09,152 DEBUG SenderThread:19614 [sender.py:send():382] send: metric -2024-02-14 11:54:09,153 DEBUG SenderThread:19614 [sender.py:send():382] send: telemetry -2024-02-14 11:54:09,153 DEBUG SenderThread:19614 [sender.py:send():382] send: metric -2024-02-14 11:54:09,153 WARNING SenderThread:19614 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) -2024-02-14 11:54:09,153 DEBUG SenderThread:19614 [sender.py:send():382] send: telemetry -2024-02-14 11:54:09,377 INFO wandb-upload_0:19614 [upload_job.py:push():131] Uploaded file /tmp/tmp5ahebo6hwandb/g03hw9hk-wandb-metadata.json -2024-02-14 11:54:09,757 INFO Thread-12 :19614 [dir_watcher.py:_on_file_created():271] file/dir created: /content/wandb/run-20240214_115405-518dheci/files/requirements.txt -2024-02-14 11:54:09,758 INFO Thread-12 :19614 [dir_watcher.py:_on_file_created():271] file/dir created: /content/wandb/run-20240214_115405-518dheci/files/wandb-metadata.json -2024-02-14 11:54:11,153 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:54:16,154 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:54:21,155 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:54:23,936 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 11:54:23,936 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 11:54:23,977 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 11:54:27,087 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:54:32,088 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:54:37,092 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:54:37,766 INFO Thread-12 :19614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_115405-518dheci/files/config.yaml -2024-02-14 11:54:38,936 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 11:54:38,937 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 11:54:38,977 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 11:54:43,054 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:54:48,055 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:54:53,056 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:54:53,936 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 11:54:53,937 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 11:54:53,977 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 11:54:58,107 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:55:03,108 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:55:08,109 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:55:08,904 DEBUG SystemMonitor:19614 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-02-14 11:55:08,905 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 11:55:08,936 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 11:55:08,937 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 11:55:08,977 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 11:55:14,040 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:55:19,041 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:55:23,936 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 11:55:23,937 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 11:55:23,977 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 11:55:24,075 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:55:29,076 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:55:34,077 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:55:38,906 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 11:55:38,936 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 11:55:38,937 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 11:55:38,977 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 11:55:39,086 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:55:40,270 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: partial_history -2024-02-14 11:55:40,272 DEBUG SenderThread:19614 [sender.py:send():382] send: metric -2024-02-14 11:55:40,272 DEBUG SenderThread:19614 [sender.py:send():382] send: metric -2024-02-14 11:55:40,272 DEBUG SenderThread:19614 [sender.py:send():382] send: metric -2024-02-14 11:55:40,272 DEBUG SenderThread:19614 [sender.py:send():382] send: history -2024-02-14 11:55:40,273 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: summary_record -2024-02-14 11:55:40,273 INFO SenderThread:19614 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-02-14 11:55:40,785 INFO Thread-12 :19614 [dir_watcher.py:_on_file_created():271] file/dir created: /content/wandb/run-20240214_115405-518dheci/files/wandb-summary.json -2024-02-14 11:55:44,274 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:55:49,275 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:55:53,937 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 11:55:53,937 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 11:55:53,938 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 11:55:55,029 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:56:00,030 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:56:05,031 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:56:08,906 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 11:56:08,937 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 11:56:08,937 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 11:56:08,977 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 11:56:10,055 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:56:10,794 INFO Thread-12 :19614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_115405-518dheci/files/config.yaml -2024-02-14 11:56:15,207 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:56:20,208 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:56:23,937 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 11:56:23,937 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 11:56:23,978 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 11:56:26,017 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:56:31,018 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:56:36,019 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:56:38,907 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 11:56:38,937 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 11:56:38,938 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 11:56:38,979 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 11:56:42,012 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:56:47,013 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:56:52,014 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:56:53,937 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 11:56:53,938 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 11:56:53,978 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 11:56:58,008 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:57:03,009 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:57:08,010 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:57:08,908 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 11:57:08,937 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 11:57:08,938 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 11:57:08,978 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 11:57:13,028 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:57:16,231 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: partial_history -2024-02-14 11:57:16,232 DEBUG SenderThread:19614 [sender.py:send():382] send: history -2024-02-14 11:57:16,233 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: summary_record -2024-02-14 11:57:16,233 INFO SenderThread:19614 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-02-14 11:57:16,816 INFO Thread-12 :19614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_115405-518dheci/files/wandb-summary.json -2024-02-14 11:57:18,234 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:57:23,235 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:57:23,937 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 11:57:23,938 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 11:57:23,940 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 11:57:29,003 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:57:34,004 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:57:38,909 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 11:57:38,938 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 11:57:38,938 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 11:57:38,978 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 11:57:39,025 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:57:44,026 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:57:49,027 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:57:53,938 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 11:57:53,938 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 11:57:53,978 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 11:57:54,048 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:57:59,049 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:58:04,050 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:58:08,909 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 11:58:08,938 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 11:58:08,938 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 11:58:08,979 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 11:58:10,044 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:58:15,045 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:58:20,046 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:58:23,938 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 11:58:23,939 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 11:58:23,979 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 11:58:26,025 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:58:31,026 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:58:36,026 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:58:38,910 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 11:58:38,938 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 11:58:38,939 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 11:58:38,979 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 11:58:41,035 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:58:46,036 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:58:48,839 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: partial_history -2024-02-14 11:58:48,840 DEBUG SenderThread:19614 [sender.py:send():382] send: history -2024-02-14 11:58:48,840 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: summary_record -2024-02-14 11:58:48,841 INFO SenderThread:19614 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-02-14 11:58:48,846 INFO Thread-12 :19614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_115405-518dheci/files/wandb-summary.json -2024-02-14 11:58:51,841 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:58:53,938 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 11:58:53,939 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 11:58:53,942 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 11:58:57,040 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:59:02,041 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:59:07,042 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:59:08,911 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 11:59:08,938 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 11:59:08,939 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 11:59:08,979 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 11:59:12,044 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:59:17,045 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:59:22,046 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:59:23,939 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 11:59:23,939 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 11:59:23,979 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 11:59:27,092 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:59:32,093 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:59:37,094 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:59:38,913 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 11:59:38,939 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 11:59:38,939 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 11:59:38,979 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 11:59:43,011 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:59:48,011 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:59:53,012 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 11:59:53,939 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 11:59:53,939 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 11:59:53,980 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 11:59:58,048 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:00:03,049 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:00:08,050 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:00:08,913 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:00:08,939 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:00:08,939 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:00:08,980 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:00:14,016 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:00:19,017 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:00:23,375 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: partial_history -2024-02-14 12:00:23,376 DEBUG SenderThread:19614 [sender.py:send():382] send: history -2024-02-14 12:00:23,377 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: summary_record -2024-02-14 12:00:23,377 INFO SenderThread:19614 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-02-14 12:00:23,876 INFO Thread-12 :19614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_115405-518dheci/files/wandb-summary.json -2024-02-14 12:00:24,043 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:00:24,044 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:00:24,044 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:00:24,148 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:00:29,149 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:00:34,150 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:00:38,913 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:00:39,042 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:00:39,042 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:00:39,043 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:00:40,133 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:00:45,134 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:00:50,135 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:00:54,042 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:00:54,043 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:00:54,043 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:00:55,142 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:01:00,143 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:01:05,144 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:01:08,914 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:01:09,042 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:01:09,043 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:01:09,043 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:01:10,196 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:01:15,197 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:01:20,198 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:01:24,043 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:01:24,043 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:01:24,044 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:01:26,181 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:01:31,182 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:01:36,183 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:01:38,915 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:01:39,043 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:01:39,043 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:01:39,044 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:01:42,174 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:01:47,175 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:01:52,176 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:01:54,043 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:01:54,043 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:01:54,044 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:01:57,193 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:01:59,452 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: partial_history -2024-02-14 12:01:59,453 DEBUG SenderThread:19614 [sender.py:send():382] send: history -2024-02-14 12:01:59,453 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: summary_record -2024-02-14 12:01:59,454 INFO SenderThread:19614 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-02-14 12:01:59,907 INFO Thread-12 :19614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_115405-518dheci/files/wandb-summary.json -2024-02-14 12:02:02,455 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:02:07,456 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:02:08,915 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:02:09,043 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:02:09,044 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:02:09,044 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:02:13,141 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:02:18,142 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:02:23,143 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:02:24,043 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:02:24,044 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:02:24,044 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:02:28,165 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:02:33,165 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:02:38,166 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:02:38,916 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:02:39,043 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:02:39,044 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:02:39,044 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:02:43,188 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:02:48,189 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:02:53,190 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:02:54,043 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:02:54,044 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:02:54,044 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:02:59,109 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:03:04,110 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:03:08,917 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:03:09,044 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:03:09,044 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:03:09,044 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:03:09,186 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:03:14,186 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:03:19,187 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:03:24,044 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:03:24,045 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:03:24,045 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:03:25,122 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:03:30,123 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:03:35,124 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:03:38,438 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: partial_history -2024-02-14 12:03:38,439 DEBUG SenderThread:19614 [sender.py:send():382] send: history -2024-02-14 12:03:38,440 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: summary_record -2024-02-14 12:03:38,440 INFO SenderThread:19614 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-02-14 12:03:38,918 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:03:38,940 INFO Thread-12 :19614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_115405-518dheci/files/wandb-summary.json -2024-02-14 12:03:39,080 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:03:39,081 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:03:39,083 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:03:40,162 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:03:45,163 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:03:50,164 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:03:54,080 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:03:54,080 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:03:54,120 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:03:55,185 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:04:00,186 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:04:05,187 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:04:08,919 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:04:09,080 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:04:09,080 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:04:09,121 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:04:10,240 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:04:15,241 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:04:20,242 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:04:24,080 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:04:24,080 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:04:24,121 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:04:25,265 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:04:30,266 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:04:35,267 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:04:38,919 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:04:39,080 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:04:39,081 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:04:39,121 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:04:41,222 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:04:46,222 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:04:51,223 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:04:54,080 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:04:54,081 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:04:54,121 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:04:56,233 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:05:01,233 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:05:06,234 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:05:08,920 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:05:09,080 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:05:09,081 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:05:09,121 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:05:12,154 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:05:15,115 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: partial_history -2024-02-14 12:05:15,116 DEBUG SenderThread:19614 [sender.py:send():382] send: history -2024-02-14 12:05:15,116 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: summary_record -2024-02-14 12:05:15,116 INFO SenderThread:19614 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-02-14 12:05:15,972 INFO Thread-12 :19614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_115405-518dheci/files/wandb-summary.json -2024-02-14 12:05:18,117 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:05:23,118 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:05:24,081 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:05:24,081 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:05:24,083 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:05:28,213 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:05:33,214 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:05:38,214 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:05:38,920 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:05:39,081 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:05:39,081 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:05:39,122 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:05:44,186 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:05:49,187 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:05:54,081 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:05:54,081 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:05:54,122 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:05:54,241 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:05:59,242 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:06:04,243 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:06:08,921 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:06:09,081 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:06:09,081 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:06:09,122 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:06:10,209 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:06:15,210 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:06:20,211 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:06:24,081 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:06:24,082 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:06:24,123 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:06:26,175 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:06:31,176 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:06:36,176 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:06:38,922 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:06:39,081 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:06:39,082 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:06:39,122 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:06:42,158 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:06:47,159 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:06:47,360 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: partial_history -2024-02-14 12:06:47,361 DEBUG SenderThread:19614 [sender.py:send():382] send: history -2024-02-14 12:06:47,362 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: summary_record -2024-02-14 12:06:47,362 INFO SenderThread:19614 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-02-14 12:06:48,002 INFO Thread-12 :19614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_115405-518dheci/files/wandb-summary.json -2024-02-14 12:06:52,363 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:06:54,081 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:06:54,082 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:06:54,084 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:06:58,137 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:07:03,138 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:07:08,139 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:07:08,923 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:07:09,082 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:07:09,082 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:07:09,122 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:07:13,144 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:07:18,145 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:07:23,146 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:07:24,082 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:07:24,082 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:07:24,122 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:07:28,241 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:07:33,242 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:07:38,243 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:07:38,923 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:07:39,082 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:07:39,082 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:07:39,123 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:07:44,199 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:07:49,200 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:07:54,082 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:07:54,083 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:07:54,123 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:07:55,200 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:08:00,201 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:08:05,202 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:08:08,924 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:08:09,082 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:08:09,083 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:08:09,123 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:08:10,227 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:08:15,228 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:08:20,229 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:08:22,698 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: partial_history -2024-02-14 12:08:22,699 DEBUG SenderThread:19614 [sender.py:send():382] send: history -2024-02-14 12:08:22,699 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: summary_record -2024-02-14 12:08:22,700 INFO SenderThread:19614 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-02-14 12:08:23,031 INFO Thread-12 :19614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_115405-518dheci/files/wandb-summary.json -2024-02-14 12:08:24,086 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:08:24,098 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:08:24,098 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:08:25,252 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:08:30,252 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:08:35,253 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:08:38,925 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:08:39,086 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:08:39,086 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:08:39,087 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:08:41,191 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:08:46,192 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:08:51,193 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:08:54,086 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:08:54,086 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:08:54,087 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:08:56,214 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:09:01,215 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:09:06,216 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:09:08,925 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:09:09,086 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:09:09,086 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:09:09,087 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:09:12,203 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:09:17,204 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:09:22,205 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:09:24,086 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:09:24,087 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:09:24,087 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:09:28,176 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:09:33,176 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:09:38,177 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:09:38,926 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:09:39,086 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:09:39,087 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:09:39,087 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:09:43,185 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:09:48,186 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:09:53,187 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:09:54,086 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:09:54,087 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:09:54,087 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:09:57,588 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: partial_history -2024-02-14 12:09:57,589 DEBUG SenderThread:19614 [sender.py:send():382] send: history -2024-02-14 12:09:57,589 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: summary_record -2024-02-14 12:09:57,590 INFO SenderThread:19614 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-02-14 12:09:58,064 INFO Thread-12 :19614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_115405-518dheci/files/wandb-summary.json -2024-02-14 12:09:58,590 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:10:03,591 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:10:08,592 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:10:08,934 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:10:09,086 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:10:09,087 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:10:09,087 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:10:14,237 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:10:19,238 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:10:24,087 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:10:24,087 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:10:24,087 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:10:24,252 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:10:29,253 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:10:34,254 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:10:38,927 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:10:39,087 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:10:39,087 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:10:39,087 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:10:40,166 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:10:45,166 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:10:50,167 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:10:54,087 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:10:54,087 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:10:54,088 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:10:55,229 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:11:00,230 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:11:05,231 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:11:08,930 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:11:09,087 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:11:09,088 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:11:09,088 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:11:10,250 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:11:15,251 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:11:20,252 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:11:24,087 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:11:24,088 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:11:24,088 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:11:26,206 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:11:31,207 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:11:36,208 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:11:37,608 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: partial_history -2024-02-14 12:11:37,609 DEBUG SenderThread:19614 [sender.py:send():382] send: history -2024-02-14 12:11:37,609 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: summary_record -2024-02-14 12:11:37,610 INFO SenderThread:19614 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-02-14 12:11:38,095 INFO Thread-12 :19614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_115405-518dheci/files/wandb-summary.json -2024-02-14 12:11:38,929 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:11:39,087 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:11:39,088 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:11:39,088 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:11:42,149 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:11:47,150 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:11:52,152 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:11:54,088 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:11:54,088 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:11:54,089 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:11:57,176 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:12:02,176 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:12:07,177 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:12:08,930 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:12:09,088 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:12:09,088 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:12:09,088 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:12:12,212 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:12:17,213 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:12:22,214 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:12:24,088 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:12:24,089 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:12:24,089 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:12:28,201 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:12:33,202 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:12:38,203 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:12:38,931 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:12:39,088 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:12:39,089 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:12:39,089 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:12:43,240 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:12:48,241 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:12:53,242 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:12:54,088 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:12:54,089 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:12:54,089 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:12:59,186 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:13:04,187 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:13:08,932 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:13:09,088 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:13:09,089 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:13:09,089 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:13:09,248 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:13:09,811 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: partial_history -2024-02-14 12:13:09,812 DEBUG SenderThread:19614 [sender.py:send():382] send: history -2024-02-14 12:13:09,812 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: summary_record -2024-02-14 12:13:09,813 INFO SenderThread:19614 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-02-14 12:13:10,124 INFO Thread-12 :19614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_115405-518dheci/files/wandb-summary.json -2024-02-14 12:13:14,814 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:13:19,815 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:13:24,089 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:13:24,089 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:13:24,089 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:13:25,186 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:13:30,187 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:13:35,188 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:13:38,932 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:13:39,089 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:13:39,089 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:13:39,089 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:13:41,169 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:13:46,170 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:13:51,171 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:13:54,089 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:13:54,089 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:13:54,090 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:13:56,205 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:14:01,206 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:14:06,207 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:14:08,933 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:14:09,089 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:14:09,089 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:14:09,090 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:14:11,234 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:14:16,235 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:14:21,235 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:14:24,089 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:14:24,089 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:14:24,090 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:14:26,238 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:14:31,238 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:14:36,239 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:14:38,934 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:14:39,089 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:14:39,089 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:14:39,090 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:14:42,155 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:14:45,618 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: partial_history -2024-02-14 12:14:45,619 DEBUG SenderThread:19614 [sender.py:send():382] send: history -2024-02-14 12:14:45,619 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: summary_record -2024-02-14 12:14:45,619 INFO SenderThread:19614 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-02-14 12:14:46,153 INFO Thread-12 :19614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_115405-518dheci/files/wandb-summary.json -2024-02-14 12:14:47,620 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:14:52,621 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:14:54,090 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:14:54,090 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:14:54,090 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:14:58,260 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:15:03,260 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:15:08,261 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:15:08,935 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:15:09,090 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:15:09,091 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:15:09,131 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:15:14,244 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:15:19,245 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:15:24,090 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:15:24,090 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:15:24,090 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:15:25,186 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:15:30,188 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:15:35,189 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:15:38,935 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:15:39,090 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:15:39,090 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:15:39,091 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:15:41,150 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:15:46,151 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:15:51,152 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:15:54,090 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:15:54,090 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:15:54,091 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:15:56,225 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:16:01,225 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:16:06,226 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:16:08,936 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:16:09,090 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:16:09,090 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:16:09,091 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:16:12,182 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:16:17,183 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:16:22,184 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:16:22,376 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: partial_history -2024-02-14 12:16:22,377 DEBUG SenderThread:19614 [sender.py:send():382] send: history -2024-02-14 12:16:22,377 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: summary_record -2024-02-14 12:16:22,378 INFO SenderThread:19614 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-02-14 12:16:23,183 INFO Thread-12 :19614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_115405-518dheci/files/wandb-summary.json -2024-02-14 12:16:24,090 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:16:24,090 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:16:24,091 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:16:28,182 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:16:33,183 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:16:38,184 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:16:38,936 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:16:39,091 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:16:39,091 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:16:39,091 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:16:43,244 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:16:48,244 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:16:53,245 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:16:54,090 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:16:54,091 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:16:54,131 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:16:59,177 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:17:04,178 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:17:08,937 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:17:09,091 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:17:09,091 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:17:09,131 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:17:09,202 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:17:14,203 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:17:19,204 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:17:24,091 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:17:24,091 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:17:24,092 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:17:25,160 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:17:30,161 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:17:35,162 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:17:38,938 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:17:39,091 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:17:39,091 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:17:39,132 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:17:40,238 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:17:45,239 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:17:50,239 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:17:54,091 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:17:54,091 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:17:54,092 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:17:54,908 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: partial_history -2024-02-14 12:17:54,909 DEBUG SenderThread:19614 [sender.py:send():382] send: history -2024-02-14 12:17:54,909 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: summary_record -2024-02-14 12:17:54,909 INFO SenderThread:19614 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-02-14 12:17:55,211 INFO Thread-12 :19614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_115405-518dheci/files/wandb-summary.json -2024-02-14 12:17:55,910 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:18:00,911 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:18:05,912 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:18:08,939 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:18:09,091 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:18:09,091 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:18:09,092 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:18:11,243 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:18:16,244 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:18:21,245 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:18:24,091 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:18:24,092 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:18:24,092 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:18:27,174 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:18:32,175 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:18:37,176 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:18:38,939 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:18:39,092 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:18:39,092 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:18:39,092 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:18:43,157 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:18:48,158 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:18:53,159 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:18:54,092 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:18:54,092 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:18:54,092 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:18:58,220 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:19:03,221 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:19:08,222 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:19:08,940 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:19:09,092 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:19:09,092 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:19:09,092 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:19:13,245 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:19:18,246 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:19:23,247 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:19:24,092 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:19:24,092 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:19:24,093 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:19:28,249 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:19:33,250 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:19:33,277 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: partial_history -2024-02-14 12:19:33,278 DEBUG SenderThread:19614 [sender.py:send():382] send: history -2024-02-14 12:19:33,278 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: summary_record -2024-02-14 12:19:33,279 INFO SenderThread:19614 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-02-14 12:19:34,239 INFO Thread-12 :19614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_115405-518dheci/files/wandb-summary.json -2024-02-14 12:19:38,280 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:19:38,941 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:19:39,092 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:19:39,092 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:19:39,093 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:19:44,218 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:19:49,218 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:19:54,092 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:19:54,092 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:19:54,133 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:19:54,242 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:19:59,243 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:20:04,244 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:20:08,942 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:20:09,092 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:20:09,093 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:20:09,133 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:20:10,235 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:20:15,236 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:20:20,236 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:20:24,092 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:20:24,093 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:20:24,093 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:20:25,241 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:20:30,242 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:20:35,243 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:20:38,942 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:20:39,093 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:20:39,093 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:20:39,093 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:20:41,154 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:20:46,155 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:20:51,156 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:20:54,093 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:20:54,093 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:20:54,093 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:20:56,202 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:21:01,203 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:21:06,203 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:21:08,943 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:21:09,007 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: partial_history -2024-02-14 12:21:09,008 DEBUG SenderThread:19614 [sender.py:send():382] send: history -2024-02-14 12:21:09,008 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: summary_record -2024-02-14 12:21:09,009 INFO SenderThread:19614 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-02-14 12:21:09,092 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:21:09,093 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:21:09,094 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:21:09,265 INFO Thread-12 :19614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_115405-518dheci/files/wandb-summary.json -2024-02-14 12:21:11,231 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:21:16,231 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:21:21,232 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:21:24,093 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:21:24,093 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:21:24,094 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:21:27,212 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:21:32,213 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:21:37,214 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:21:38,944 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:21:39,093 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:21:39,094 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:21:39,094 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:21:43,168 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:21:48,169 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:21:53,170 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:21:54,094 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:21:54,094 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:21:54,094 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:21:58,230 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:22:03,230 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:22:08,231 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:22:08,945 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:22:09,094 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:22:09,094 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:22:09,094 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:22:14,179 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:22:19,180 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:22:24,094 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:22:24,094 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:22:24,094 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:22:24,245 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:22:29,245 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:22:34,246 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:22:38,945 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:22:39,094 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:22:39,094 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:22:39,095 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:22:40,193 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:22:41,724 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: partial_history -2024-02-14 12:22:41,725 DEBUG SenderThread:19614 [sender.py:send():382] send: history -2024-02-14 12:22:41,725 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: summary_record -2024-02-14 12:22:41,726 INFO SenderThread:19614 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-02-14 12:22:42,292 INFO Thread-12 :19614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_115405-518dheci/files/wandb-summary.json -2024-02-14 12:22:45,727 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:22:46,046 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: partial_history -2024-02-14 12:22:46,047 DEBUG SenderThread:19614 [sender.py:send():382] send: metric -2024-02-14 12:22:46,048 DEBUG SenderThread:19614 [sender.py:send():382] send: metric -2024-02-14 12:22:46,048 DEBUG SenderThread:19614 [sender.py:send():382] send: metric -2024-02-14 12:22:46,048 DEBUG SenderThread:19614 [sender.py:send():382] send: metric -2024-02-14 12:22:46,048 DEBUG SenderThread:19614 [sender.py:send():382] send: metric -2024-02-14 12:22:46,048 DEBUG SenderThread:19614 [sender.py:send():382] send: history -2024-02-14 12:22:46,048 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: summary_record -2024-02-14 12:22:46,049 INFO SenderThread:19614 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-02-14 12:22:46,293 INFO Thread-12 :19614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_115405-518dheci/files/wandb-summary.json -2024-02-14 12:22:51,050 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:22:52,245 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: pause -2024-02-14 12:22:52,245 INFO HandlerThread:19614 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-02-14 12:22:52,245 INFO HandlerThread:19614 [system_monitor.py:finish():203] Stopping system monitor -2024-02-14 12:22:52,245 DEBUG SystemMonitor:19614 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-02-14 12:22:52,246 DEBUG SystemMonitor:19614 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-02-14 12:22:52,246 INFO HandlerThread:19614 [interfaces.py:finish():202] Joined cpu monitor -2024-02-14 12:22:52,247 INFO HandlerThread:19614 [interfaces.py:finish():202] Joined disk monitor -2024-02-14 12:22:52,254 INFO HandlerThread:19614 [interfaces.py:finish():202] Joined gpu monitor -2024-02-14 12:22:52,254 INFO HandlerThread:19614 [interfaces.py:finish():202] Joined memory monitor -2024-02-14 12:22:52,254 INFO HandlerThread:19614 [interfaces.py:finish():202] Joined network monitor -2024-02-14 12:22:52,255 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:22:54,094 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:22:54,094 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:22:54,094 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:22:56,215 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:23:01,216 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:23:06,221 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:23:07,323 INFO Thread-12 :19614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_115405-518dheci/files/config.yaml -2024-02-14 12:23:09,094 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:23:09,094 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:23:09,095 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:23:12,274 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:23:17,275 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:23:22,276 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:23:24,094 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:23:24,095 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:23:24,095 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:23:28,181 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:23:33,182 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:23:38,184 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:23:39,095 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:23:39,095 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:23:39,095 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:23:44,165 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:23:49,166 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:23:54,095 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:23:54,095 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:23:54,095 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:23:54,249 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:23:59,250 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:24:04,251 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:24:09,095 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:24:09,095 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:24:09,096 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:24:10,244 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:24:15,244 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:24:20,245 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:24:24,095 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:24:24,095 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:24:24,096 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:24:26,189 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:24:31,190 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:24:36,191 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:24:39,095 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:24:39,096 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:24:39,096 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:24:41,200 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:24:46,201 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:24:51,202 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:24:54,095 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:24:54,096 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:24:54,096 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:24:57,166 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:25:02,167 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:25:07,168 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:25:09,096 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:25:09,096 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:25:09,096 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:25:12,201 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:25:17,202 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:25:22,203 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:25:24,096 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:25:24,096 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:25:24,097 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:25:28,169 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:25:33,170 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:25:38,171 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:25:39,096 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:25:39,096 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:25:39,097 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:25:43,248 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:25:48,249 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:25:53,250 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:25:54,096 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:25:54,096 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:25:54,097 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:25:59,197 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:26:04,198 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:26:09,096 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:26:09,096 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:26:09,097 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:26:09,208 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:26:14,209 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:26:19,210 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:26:24,096 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:26:24,097 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:26:24,097 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:26:24,244 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:26:29,245 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:26:34,246 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:26:39,096 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:26:39,097 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:26:39,097 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:26:40,214 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:26:45,215 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:26:50,216 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:26:54,097 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:26:54,097 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:26:54,097 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:26:55,237 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:27:00,238 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:27:05,239 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:27:09,097 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:27:09,097 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:27:09,098 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:27:11,238 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:27:16,239 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:27:21,240 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:27:24,097 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:27:24,097 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:27:24,098 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:27:27,154 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:27:32,155 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:27:37,156 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:27:39,097 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:27:39,097 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:27:39,098 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:27:42,263 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:27:47,264 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:27:52,265 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:27:54,097 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:27:54,097 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:27:54,098 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:27:58,182 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:28:03,183 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:28:08,184 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:28:09,097 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:28:09,098 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:28:09,098 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:28:13,242 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:28:18,243 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:28:23,244 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:28:24,098 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:28:24,098 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:28:24,098 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:28:28,254 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:28:33,255 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:28:38,256 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:28:39,098 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:28:39,098 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:28:39,099 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:28:44,249 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:28:49,250 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:28:54,098 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:28:54,098 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:28:54,099 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:28:55,242 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:29:00,243 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:29:05,244 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:29:09,098 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:29:09,099 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:29:09,099 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:29:10,249 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:29:15,250 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:29:20,251 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:29:24,098 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:29:24,099 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:29:24,099 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:29:26,154 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:29:31,155 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:29:36,156 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:29:39,099 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:29:39,099 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:29:39,099 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:29:41,204 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:29:46,205 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:29:51,206 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:29:54,099 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:29:54,099 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:29:54,099 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:29:56,225 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:30:01,226 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:30:06,228 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:30:09,099 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:30:09,099 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:30:09,099 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:30:12,210 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:30:17,211 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:30:22,212 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:30:24,099 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:30:24,099 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:30:24,100 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:30:28,192 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:30:33,193 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:30:38,194 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:30:39,099 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:30:39,099 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:30:39,100 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:30:44,173 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:30:49,173 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:30:54,099 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:30:54,100 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:30:54,100 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:30:54,213 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:30:59,214 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:31:04,215 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:31:09,100 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:31:09,100 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:31:09,100 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:31:10,185 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:31:15,186 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:31:20,187 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:31:24,100 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:31:24,100 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:31:24,100 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:31:26,184 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:31:31,185 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:31:36,186 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:31:39,100 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:31:39,100 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:31:39,101 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:31:41,236 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:31:46,237 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:31:51,238 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:31:54,100 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:31:54,100 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:31:54,101 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:31:57,220 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:32:02,221 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:32:07,222 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:32:09,100 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:32:09,100 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:32:09,101 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:32:13,211 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:32:18,211 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:32:23,213 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:32:24,100 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:32:24,101 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:32:24,101 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:32:28,225 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:32:33,226 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:32:38,227 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:32:39,101 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:32:39,101 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:32:39,101 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:32:44,181 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:32:49,182 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:32:54,101 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:32:54,101 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:32:54,101 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:32:55,172 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:33:00,174 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:33:05,174 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:33:09,101 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:33:09,101 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:33:09,101 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:33:10,187 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:33:15,188 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:33:20,189 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:33:24,101 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:33:24,101 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:33:24,102 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:33:25,231 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:33:30,232 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:33:35,233 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:33:39,101 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:33:39,101 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:33:39,102 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:33:40,253 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:33:45,253 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:33:50,255 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:33:54,102 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:33:54,102 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:33:54,102 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:33:56,233 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:34:01,234 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:34:06,235 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:34:09,102 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:34:09,102 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:34:09,102 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:34:12,195 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:34:17,196 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:34:22,197 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:34:24,102 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:34:24,102 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:34:24,103 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:34:28,182 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:34:33,183 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:34:38,184 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:34:39,102 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:34:39,102 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:34:39,102 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:34:43,253 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:34:48,254 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:34:53,255 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:34:54,102 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:34:54,102 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:34:54,103 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:34:59,254 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:35:04,255 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:35:09,102 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:35:09,102 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:35:09,103 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:35:09,257 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:35:14,258 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:35:19,259 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:35:24,103 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:35:24,103 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:35:24,103 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:35:25,173 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:35:30,174 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:35:35,175 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:35:39,103 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:35:39,103 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:35:39,103 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:35:40,269 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:35:45,270 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:35:50,271 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:35:54,103 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:35:54,103 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:35:54,103 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:35:56,219 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:36:01,220 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:36:06,221 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:36:09,103 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:36:09,103 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:36:09,104 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:36:12,201 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:36:17,203 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:36:22,204 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:36:24,103 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:36:24,103 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:36:24,104 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:36:28,184 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:36:33,185 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:36:38,186 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:36:39,103 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:36:39,103 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:36:39,104 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:36:43,271 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:36:48,273 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:36:53,274 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:36:54,103 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:36:54,104 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:36:54,144 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:36:59,198 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:37:04,199 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:37:09,104 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:37:09,104 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:37:09,104 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:37:10,195 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:37:15,196 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:37:20,197 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:37:24,104 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:37:24,104 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:37:24,104 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:37:25,242 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:37:30,243 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:37:35,244 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:37:39,104 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:37:39,104 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:37:39,105 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:37:40,265 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:37:45,266 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:37:50,267 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:37:54,104 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:37:54,104 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:37:54,105 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:37:56,254 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:38:01,255 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:38:06,256 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:38:09,104 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:38:09,104 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:38:09,105 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:38:12,243 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:38:17,245 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:38:22,246 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:38:24,105 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:38:24,105 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:38:24,105 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:38:28,193 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:38:33,194 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:38:38,195 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:38:39,105 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:38:39,105 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:38:39,105 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:38:44,193 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:38:49,194 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:38:54,105 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:38:54,105 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:38:54,106 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:38:54,236 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:38:59,237 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:39:04,238 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:39:09,105 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:39:09,105 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:39:09,106 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:39:09,255 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:39:14,256 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:39:19,257 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:39:24,105 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:39:24,105 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:39:24,106 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:39:25,216 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:39:30,217 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:39:35,218 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:39:39,105 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:39:39,106 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:39:39,106 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:39:40,256 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:39:45,257 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:39:50,258 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:39:54,105 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:39:54,106 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:39:54,106 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:39:56,174 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:40:01,175 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:40:06,176 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:40:09,105 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:40:09,106 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:40:09,106 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:40:12,172 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:40:17,173 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:40:22,174 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:40:24,105 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:40:24,106 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:40:24,106 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:40:27,185 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:40:32,186 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:40:37,187 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:40:39,106 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:40:39,106 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:40:39,106 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:40:42,258 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:40:47,259 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:40:52,260 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:40:54,106 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:40:54,106 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:40:54,107 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:40:58,254 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:41:03,255 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:41:08,256 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:41:09,106 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:41:09,106 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:41:09,107 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:41:13,269 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:41:18,270 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:41:23,271 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:41:24,106 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:41:24,107 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:41:24,107 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:41:29,233 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:41:34,234 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:41:39,106 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:41:39,107 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:41:39,107 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:41:39,257 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:41:44,258 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:41:49,259 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:41:54,106 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:41:54,107 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:41:54,107 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:41:55,257 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:42:00,258 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:42:05,259 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:42:09,107 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:42:09,107 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:42:09,107 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:42:10,266 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:42:15,267 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:42:20,268 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:42:24,107 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:42:24,107 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:42:24,108 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:42:26,221 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:42:31,221 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:42:36,222 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:42:39,107 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:42:39,107 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:42:39,108 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:42:41,277 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:42:46,278 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:42:51,279 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:42:54,107 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:42:54,107 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:42:54,108 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:42:57,228 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:43:02,229 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:43:07,230 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:43:09,107 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:43:09,108 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:43:09,108 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:43:13,193 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:43:18,194 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:43:23,195 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:43:24,107 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:43:24,108 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:43:24,108 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:43:29,184 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:43:34,185 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:43:39,107 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:43:39,108 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:43:39,108 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:43:39,186 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:43:44,187 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:43:49,188 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:43:54,108 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:43:54,108 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:43:54,108 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:43:54,216 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:43:59,217 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:44:04,217 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:44:09,108 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:44:09,108 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:44:09,108 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:44:10,204 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:44:15,204 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:44:20,206 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:44:24,108 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:44:24,108 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:44:24,109 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:44:25,258 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:44:30,259 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:44:35,261 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:44:39,108 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:44:39,108 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:44:39,109 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:44:41,259 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:44:46,260 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:44:51,261 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:44:54,108 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:44:54,108 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:44:54,109 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:44:57,243 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:45:02,244 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:45:07,245 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:45:09,108 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:45:09,109 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:45:09,109 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:45:12,268 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:45:17,269 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:45:22,270 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:45:24,108 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:45:24,109 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:45:24,109 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:45:28,249 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:45:33,249 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:45:38,250 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:45:39,109 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:45:39,109 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:45:39,109 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:45:44,224 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:45:49,225 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:45:54,109 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:45:54,109 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:45:54,109 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:45:55,220 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:46:00,221 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:46:05,222 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:46:09,109 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:46:09,109 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:46:09,109 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:46:11,177 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:46:16,178 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:46:21,179 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:46:24,109 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:46:24,109 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:46:24,109 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:46:26,240 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:46:31,241 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:46:36,242 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:46:39,109 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:46:39,109 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:46:39,110 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:46:41,264 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:46:46,265 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:46:51,266 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:46:54,109 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:46:54,109 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:46:54,110 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:46:57,214 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:47:02,215 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:47:07,216 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:47:09,109 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:47:09,110 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:47:09,110 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:47:13,175 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:47:18,176 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:47:23,177 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:47:24,109 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:47:24,110 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:47:24,110 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:47:28,178 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:47:33,179 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:47:38,180 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:47:39,109 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:47:39,110 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:47:39,110 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:47:43,207 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:47:48,208 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:47:53,209 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:47:54,110 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:47:54,110 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:47:54,110 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:47:59,204 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:48:04,205 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:48:09,110 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:48:09,110 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:48:09,110 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:48:09,217 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:48:14,218 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:48:19,219 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:48:24,110 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:48:24,110 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:48:24,111 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:48:24,263 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:48:29,264 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:48:34,265 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:48:39,110 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:48:39,110 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:48:39,111 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:48:40,177 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:48:45,178 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:48:50,179 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:48:54,110 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:48:54,111 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:48:54,111 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:48:55,241 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:49:00,242 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:49:05,243 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:49:09,111 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:49:09,111 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:49:09,111 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:49:11,190 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:49:16,191 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:49:21,191 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:49:24,110 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:49:24,111 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:49:24,111 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:49:26,227 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:49:31,228 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:49:31,961 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: resume -2024-02-14 12:49:31,961 INFO HandlerThread:19614 [handler.py:handle_request_resume():699] starting system metrics thread -2024-02-14 12:49:31,961 INFO HandlerThread:19614 [system_monitor.py:start():194] Starting system monitor -2024-02-14 12:49:31,961 INFO SystemMonitor:19614 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-02-14 12:49:31,961 INFO SystemMonitor:19614 [interfaces.py:start():190] Started cpu monitoring -2024-02-14 12:49:31,962 INFO SystemMonitor:19614 [interfaces.py:start():190] Started disk monitoring -2024-02-14 12:49:31,963 INFO SystemMonitor:19614 [interfaces.py:start():190] Started gpu monitoring -2024-02-14 12:49:31,964 INFO SystemMonitor:19614 [interfaces.py:start():190] Started memory monitoring -2024-02-14 12:49:31,964 INFO SystemMonitor:19614 [interfaces.py:start():190] Started network monitoring -2024-02-14 12:49:32,001 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: pause -2024-02-14 12:49:32,001 INFO HandlerThread:19614 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-02-14 12:49:32,001 INFO HandlerThread:19614 [system_monitor.py:finish():203] Stopping system monitor -2024-02-14 12:49:32,001 DEBUG SystemMonitor:19614 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-02-14 12:49:32,001 DEBUG SystemMonitor:19614 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-02-14 12:49:32,001 DEBUG SystemMonitor:19614 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-02-14 12:49:32,002 INFO HandlerThread:19614 [interfaces.py:finish():202] Joined cpu monitor -2024-02-14 12:49:32,002 INFO HandlerThread:19614 [interfaces.py:finish():202] Joined disk monitor -2024-02-14 12:49:32,008 INFO HandlerThread:19614 [interfaces.py:finish():202] Joined gpu monitor -2024-02-14 12:49:32,008 INFO HandlerThread:19614 [interfaces.py:finish():202] Joined memory monitor -2024-02-14 12:49:32,008 INFO HandlerThread:19614 [interfaces.py:finish():202] Joined network monitor -2024-02-14 12:49:32,008 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:49:34,169 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: resume -2024-02-14 12:49:34,169 INFO HandlerThread:19614 [handler.py:handle_request_resume():699] starting system metrics thread -2024-02-14 12:49:34,169 INFO HandlerThread:19614 [system_monitor.py:start():194] Starting system monitor -2024-02-14 12:49:34,170 INFO SystemMonitor:19614 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-02-14 12:49:34,170 INFO SystemMonitor:19614 [interfaces.py:start():190] Started cpu monitoring -2024-02-14 12:49:34,171 INFO SystemMonitor:19614 [interfaces.py:start():190] Started disk monitoring -2024-02-14 12:49:34,171 INFO SystemMonitor:19614 [interfaces.py:start():190] Started gpu monitoring -2024-02-14 12:49:34,172 INFO SystemMonitor:19614 [interfaces.py:start():190] Started memory monitoring -2024-02-14 12:49:34,172 INFO SystemMonitor:19614 [interfaces.py:start():190] Started network monitoring -2024-02-14 12:49:34,433 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: pause -2024-02-14 12:49:34,433 INFO HandlerThread:19614 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-02-14 12:49:34,433 INFO HandlerThread:19614 [system_monitor.py:finish():203] Stopping system monitor -2024-02-14 12:49:34,434 DEBUG SystemMonitor:19614 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-02-14 12:49:34,434 DEBUG SystemMonitor:19614 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-02-14 12:49:34,434 DEBUG SystemMonitor:19614 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-02-14 12:49:34,435 INFO HandlerThread:19614 [interfaces.py:finish():202] Joined cpu monitor -2024-02-14 12:49:34,435 INFO HandlerThread:19614 [interfaces.py:finish():202] Joined disk monitor -2024-02-14 12:49:34,440 INFO HandlerThread:19614 [interfaces.py:finish():202] Joined gpu monitor -2024-02-14 12:49:34,441 INFO HandlerThread:19614 [interfaces.py:finish():202] Joined memory monitor -2024-02-14 12:49:34,441 INFO HandlerThread:19614 [interfaces.py:finish():202] Joined network monitor -2024-02-14 12:49:34,441 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:49:36,442 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:49:36,952 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: resume -2024-02-14 12:49:36,952 INFO HandlerThread:19614 [handler.py:handle_request_resume():699] starting system metrics thread -2024-02-14 12:49:36,952 INFO HandlerThread:19614 [system_monitor.py:start():194] Starting system monitor -2024-02-14 12:49:36,952 INFO SystemMonitor:19614 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-02-14 12:49:36,953 INFO SystemMonitor:19614 [interfaces.py:start():190] Started cpu monitoring -2024-02-14 12:49:36,953 INFO SystemMonitor:19614 [interfaces.py:start():190] Started disk monitoring -2024-02-14 12:49:36,954 INFO SystemMonitor:19614 [interfaces.py:start():190] Started gpu monitoring -2024-02-14 12:49:36,954 INFO SystemMonitor:19614 [interfaces.py:start():190] Started memory monitoring -2024-02-14 12:49:36,955 INFO SystemMonitor:19614 [interfaces.py:start():190] Started network monitoring -2024-02-14 12:49:36,958 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: pause -2024-02-14 12:49:36,958 INFO HandlerThread:19614 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-02-14 12:49:36,958 INFO HandlerThread:19614 [system_monitor.py:finish():203] Stopping system monitor -2024-02-14 12:49:36,959 INFO HandlerThread:19614 [interfaces.py:finish():202] Joined cpu monitor -2024-02-14 12:49:36,959 INFO HandlerThread:19614 [interfaces.py:finish():202] Joined disk monitor -2024-02-14 12:49:36,959 DEBUG SystemMonitor:19614 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-02-14 12:49:36,959 DEBUG SystemMonitor:19614 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-02-14 12:49:36,959 DEBUG SystemMonitor:19614 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-02-14 12:49:36,966 INFO HandlerThread:19614 [interfaces.py:finish():202] Joined gpu monitor -2024-02-14 12:49:36,966 INFO HandlerThread:19614 [interfaces.py:finish():202] Joined memory monitor -2024-02-14 12:49:36,966 INFO HandlerThread:19614 [interfaces.py:finish():202] Joined network monitor -2024-02-14 12:49:36,967 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:49:38,539 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: resume -2024-02-14 12:49:38,540 INFO HandlerThread:19614 [handler.py:handle_request_resume():699] starting system metrics thread -2024-02-14 12:49:38,540 INFO HandlerThread:19614 [system_monitor.py:start():194] Starting system monitor -2024-02-14 12:49:38,540 INFO SystemMonitor:19614 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-02-14 12:49:38,540 INFO SystemMonitor:19614 [interfaces.py:start():190] Started cpu monitoring -2024-02-14 12:49:38,541 INFO SystemMonitor:19614 [interfaces.py:start():190] Started disk monitoring -2024-02-14 12:49:38,542 INFO SystemMonitor:19614 [interfaces.py:start():190] Started gpu monitoring -2024-02-14 12:49:38,542 INFO SystemMonitor:19614 [interfaces.py:start():190] Started memory monitoring -2024-02-14 12:49:38,544 INFO SystemMonitor:19614 [interfaces.py:start():190] Started network monitoring -2024-02-14 12:49:39,006 DEBUG SenderThread:19614 [sender.py:send():382] send: config -2024-02-14 12:49:39,007 DEBUG SenderThread:19614 [sender.py:send():382] send: metric -2024-02-14 12:49:39,007 DEBUG SenderThread:19614 [sender.py:send():382] send: metric -2024-02-14 12:49:39,007 WARNING SenderThread:19614 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) -2024-02-14 12:49:39,110 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:49:39,111 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:49:39,111 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:49:42,264 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:49:47,265 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:49:52,266 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:49:54,111 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:49:54,111 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:49:54,112 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:49:58,222 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:50:03,223 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:50:08,229 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:50:08,884 INFO Thread-12 :19614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_115405-518dheci/files/config.yaml -2024-02-14 12:50:09,111 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:50:09,111 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:50:09,112 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:50:14,223 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:50:19,224 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:50:24,111 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:50:24,111 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:50:24,112 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:50:25,201 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:50:30,202 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:50:35,203 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:50:38,544 DEBUG SystemMonitor:19614 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-02-14 12:50:38,545 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:50:39,111 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:50:39,112 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:50:39,112 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:50:41,197 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:50:46,198 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:50:51,199 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:50:54,111 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:50:54,112 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:50:54,112 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status -2024-02-14 12:50:57,198 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:50:57,234 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: partial_history -2024-02-14 12:50:57,235 DEBUG SenderThread:19614 [sender.py:send():382] send: history -2024-02-14 12:50:57,236 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: summary_record -2024-02-14 12:50:57,236 INFO SenderThread:19614 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-02-14 12:50:57,900 INFO Thread-12 :19614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_115405-518dheci/files/wandb-summary.json -2024-02-14 12:50:57,901 INFO Thread-12 :19614 [dir_watcher.py:_on_file_created():271] file/dir created: /content/wandb/run-20240214_115405-518dheci/files/output.log -2024-02-14 12:50:59,901 INFO Thread-12 :19614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_115405-518dheci/files/output.log -2024-02-14 12:51:02,277 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:51:07,278 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: status_report -2024-02-14 12:51:08,563 DEBUG SenderThread:19614 [sender.py:send():382] send: stats -2024-02-14 12:51:09,112 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: internal_messages -2024-02-14 12:51:09,112 DEBUG HandlerThread:19614 [handler.py:handle_request():146] handle_request: stop_status -2024-02-14 12:51:09,112 DEBUG SenderThread:19614 [sender.py:send_request():409] send_request: stop_status +2024-02-14 12:54:29,380 INFO StreamThr :35088 [internal.py:wandb_internal():86] W&B internal server running at pid: 35088, started at: 2024-02-14 12:54:29.379470 +2024-02-14 12:54:29,381 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: status +2024-02-14 12:54:30,447 INFO WriterThread:35088 [datastore.py:open_for_write():87] open: /content/wandb/run-20240214_125429-fe3s7gan/run-fe3s7gan.wandb +2024-02-14 12:54:30,447 DEBUG SenderThread:35088 [sender.py:send():382] send: header +2024-02-14 12:54:30,450 DEBUG SenderThread:35088 [sender.py:send():382] send: run +2024-02-14 12:54:30,674 INFO SenderThread:35088 [dir_watcher.py:__init__():211] watching files in: /content/wandb/run-20240214_125429-fe3s7gan/files +2024-02-14 12:54:30,674 INFO SenderThread:35088 [sender.py:_start_run_threads():1136] run started: fe3s7gan with start time 1707915269.380721 +2024-02-14 12:54:30,681 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: check_version +2024-02-14 12:54:30,682 DEBUG SenderThread:35088 [sender.py:send_request():409] send_request: check_version +2024-02-14 12:54:30,815 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: run_start +2024-02-14 12:54:30,824 DEBUG HandlerThread:35088 [system_info.py:__init__():27] System info init +2024-02-14 12:54:30,824 DEBUG HandlerThread:35088 [system_info.py:__init__():42] System info init done +2024-02-14 12:54:30,824 INFO HandlerThread:35088 [system_monitor.py:start():194] Starting system monitor +2024-02-14 12:54:30,824 INFO SystemMonitor:35088 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-14 12:54:30,825 INFO HandlerThread:35088 [system_monitor.py:probe():214] Collecting system info +2024-02-14 12:54:30,825 INFO SystemMonitor:35088 [interfaces.py:start():190] Started cpu monitoring +2024-02-14 12:54:30,826 INFO SystemMonitor:35088 [interfaces.py:start():190] Started disk monitoring +2024-02-14 12:54:30,826 INFO SystemMonitor:35088 [interfaces.py:start():190] Started gpu monitoring +2024-02-14 12:54:30,828 INFO SystemMonitor:35088 [interfaces.py:start():190] Started memory monitoring +2024-02-14 12:54:30,828 INFO SystemMonitor:35088 [interfaces.py:start():190] Started network monitoring +2024-02-14 12:54:30,832 DEBUG HandlerThread:35088 [system_info.py:probe():151] Probing system +2024-02-14 12:54:30,833 DEBUG HandlerThread:35088 [gitlib.py:_init_repo():56] git repository is invalid +2024-02-14 12:54:30,833 DEBUG HandlerThread:35088 [system_info.py:probe():199] Probing system done +2024-02-14 12:54:30,833 DEBUG HandlerThread:35088 [system_monitor.py:probe():223] {'os': 'Linux-6.1.58+-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-02-14T12:54:30.832517', 'startedAt': '2024-02-14T12:54:29.373808', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'Smart_Agriculture_Falcon.ipynb', 'codePathLocal': None, 'colab': 'https://colab.research.google.com/notebook#fileId=1kv2VO7q0c8xX_DQt5NERLeJJZtHW6zHO', 'host': 'f4b889916ade', 'username': 'root', 'executable': '/usr/bin/python3', 'cpu_count': 4, 'cpu_count_logical': 8, 'cpu_freq': {'current': 2299.998, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2299.998, 'min': 0.0, 'max': 0.0}, {'current': 2299.998, 'min': 0.0, 'max': 0.0}, {'current': 2299.998, 'min': 0.0, 'max': 0.0}, {'current': 2299.998, 'min': 0.0, 'max': 0.0}, {'current': 2299.998, 'min': 0.0, 'max': 0.0}, {'current': 2299.998, 'min': 0.0, 'max': 0.0}, {'current': 2299.998, 'min': 0.0, 'max': 0.0}, {'current': 2299.998, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 166.77415084838867, 'used': 66.21718978881836}}, 'gpu': 'Tesla T4', 'gpu_count': 1, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 50.993690490722656}} +2024-02-14 12:54:30,834 INFO HandlerThread:35088 [system_monitor.py:probe():224] Finished collecting system info +2024-02-14 12:54:30,834 INFO HandlerThread:35088 [system_monitor.py:probe():227] Publishing system info +2024-02-14 12:54:30,834 INFO HandlerThread:35088 [system_monitor.py:probe():229] Finished publishing system info +2024-02-14 12:54:30,840 DEBUG SenderThread:35088 [sender.py:send():382] send: files +2024-02-14 12:54:30,840 INFO SenderThread:35088 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-02-14 12:54:30,856 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: python_packages +2024-02-14 12:54:30,856 DEBUG SenderThread:35088 [sender.py:send_request():409] send_request: python_packages +2024-02-14 12:54:30,896 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: stop_status +2024-02-14 12:54:30,896 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-14 12:54:30,897 DEBUG SenderThread:35088 [sender.py:send_request():409] send_request: stop_status +2024-02-14 12:54:31,103 DEBUG SenderThread:35088 [sender.py:send():382] send: telemetry +2024-02-14 12:54:31,103 DEBUG SenderThread:35088 [sender.py:send():382] send: config +2024-02-14 12:54:31,104 DEBUG SenderThread:35088 [sender.py:send():382] send: metric +2024-02-14 12:54:31,104 DEBUG SenderThread:35088 [sender.py:send():382] send: telemetry +2024-02-14 12:54:31,104 DEBUG SenderThread:35088 [sender.py:send():382] send: metric +2024-02-14 12:54:31,104 WARNING SenderThread:35088 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) +2024-02-14 12:54:31,104 DEBUG SenderThread:35088 [sender.py:send():382] send: telemetry +2024-02-14 12:54:31,178 INFO wandb-upload_0:35088 [upload_job.py:push():131] Uploaded file /tmp/tmp668dgil2wandb/pe1jrqzh-wandb-metadata.json +2024-02-14 12:54:31,675 INFO Thread-12 :35088 [dir_watcher.py:_on_file_created():271] file/dir created: /content/wandb/run-20240214_125429-fe3s7gan/files/requirements.txt +2024-02-14 12:54:31,675 INFO Thread-12 :35088 [dir_watcher.py:_on_file_created():271] file/dir created: /content/wandb/run-20240214_125429-fe3s7gan/files/wandb-metadata.json +2024-02-14 12:54:35,105 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: status_report +2024-02-14 12:54:40,106 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: status_report +2024-02-14 12:54:45,107 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: status_report +2024-02-14 12:54:45,857 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: stop_status +2024-02-14 12:54:45,857 DEBUG SenderThread:35088 [sender.py:send_request():409] send_request: stop_status +2024-02-14 12:54:45,897 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-14 12:54:50,987 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: status_report +2024-02-14 12:54:55,988 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: status_report +2024-02-14 12:55:00,857 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: stop_status +2024-02-14 12:55:00,857 DEBUG SenderThread:35088 [sender.py:send_request():409] send_request: stop_status +2024-02-14 12:55:00,897 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-14 12:55:01,012 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: status_report +2024-02-14 12:55:01,684 INFO Thread-12 :35088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_125429-fe3s7gan/files/config.yaml +2024-02-14 12:55:06,125 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: status_report +2024-02-14 12:55:11,126 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: status_report +2024-02-14 12:55:15,857 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: stop_status +2024-02-14 12:55:15,857 DEBUG SenderThread:35088 [sender.py:send_request():409] send_request: stop_status +2024-02-14 12:55:15,897 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-14 12:55:16,924 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: status_report +2024-02-14 12:55:21,925 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: status_report +2024-02-14 12:55:26,926 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: status_report +2024-02-14 12:55:30,828 DEBUG SystemMonitor:35088 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-14 12:55:30,829 DEBUG SenderThread:35088 [sender.py:send():382] send: stats +2024-02-14 12:55:30,857 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: stop_status +2024-02-14 12:55:30,857 DEBUG SenderThread:35088 [sender.py:send_request():409] send_request: stop_status +2024-02-14 12:55:30,897 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-14 12:55:31,981 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: status_report +2024-02-14 12:55:36,982 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: status_report +2024-02-14 12:55:41,983 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: status_report +2024-02-14 12:55:45,857 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: stop_status +2024-02-14 12:55:45,858 DEBUG SenderThread:35088 [sender.py:send_request():409] send_request: stop_status +2024-02-14 12:55:45,898 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-14 12:55:47,973 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: status_report +2024-02-14 12:55:51,040 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: partial_history +2024-02-14 12:55:51,042 DEBUG SenderThread:35088 [sender.py:send():382] send: metric +2024-02-14 12:55:51,042 DEBUG SenderThread:35088 [sender.py:send():382] send: metric +2024-02-14 12:55:51,042 DEBUG SenderThread:35088 [sender.py:send():382] send: metric +2024-02-14 12:55:51,042 DEBUG SenderThread:35088 [sender.py:send():382] send: history +2024-02-14 12:55:51,043 DEBUG SenderThread:35088 [sender.py:send_request():409] send_request: summary_record +2024-02-14 12:55:51,045 INFO SenderThread:35088 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-02-14 12:55:51,700 INFO Thread-12 :35088 [dir_watcher.py:_on_file_created():271] file/dir created: /content/wandb/run-20240214_125429-fe3s7gan/files/wandb-summary.json +2024-02-14 12:55:51,701 INFO Thread-12 :35088 [dir_watcher.py:_on_file_created():271] file/dir created: /content/wandb/run-20240214_125429-fe3s7gan/files/output.log +2024-02-14 12:55:53,082 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: status_report +2024-02-14 12:55:53,701 INFO Thread-12 :35088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /content/wandb/run-20240214_125429-fe3s7gan/files/output.log +2024-02-14 12:55:58,083 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: status_report +2024-02-14 12:56:00,830 DEBUG SenderThread:35088 [sender.py:send():382] send: stats +2024-02-14 12:56:00,857 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: stop_status +2024-02-14 12:56:00,858 DEBUG HandlerThread:35088 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-14 12:56:00,858 DEBUG SenderThread:35088 [sender.py:send_request():409] send_request: stop_status