diff --git "a/wandb/run-20240103_105541-kk0m5fl8/logs/debug-internal.log" "b/wandb/run-20240103_105541-kk0m5fl8/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20240103_105541-kk0m5fl8/logs/debug-internal.log" @@ -0,0 +1,1170 @@ +2024-01-03 10:55:41,500 INFO StreamThr :155 [internal.py:wandb_internal():86] W&B internal server running at pid: 155, started at: 2024-01-03 10:55:41.500183 +2024-01-03 10:55:41,502 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status +2024-01-03 10:55:42,151 INFO WriterThread:155 [datastore.py:open_for_write():85] open: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/run-kk0m5fl8.wandb +2024-01-03 10:55:42,151 DEBUG SenderThread:155 [sender.py:send():382] send: header +2024-01-03 10:55:42,166 DEBUG SenderThread:155 [sender.py:send():382] send: run +2024-01-03 10:55:42,664 INFO SenderThread:155 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files +2024-01-03 10:55:42,664 INFO SenderThread:155 [sender.py:_start_run_threads():1136] run started: kk0m5fl8 with start time 1704279341.501455 +2024-01-03 10:55:42,671 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: check_version +2024-01-03 10:55:42,672 DEBUG SenderThread:155 [sender.py:send_request():409] send_request: check_version +2024-01-03 10:55:42,731 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: run_start +2024-01-03 10:55:42,734 DEBUG HandlerThread:155 [system_info.py:__init__():32] System info init +2024-01-03 10:55:42,734 DEBUG HandlerThread:155 [system_info.py:__init__():47] System info init done +2024-01-03 10:55:42,735 INFO HandlerThread:155 [system_monitor.py:start():194] Starting system monitor +2024-01-03 10:55:42,735 INFO SystemMonitor:155 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-01-03 10:55:42,735 INFO SystemMonitor:155 [interfaces.py:start():190] Started cpu monitoring +2024-01-03 10:55:42,736 INFO SystemMonitor:155 [interfaces.py:start():190] Started disk monitoring +2024-01-03 10:55:42,736 INFO HandlerThread:155 [system_monitor.py:probe():214] Collecting system info +2024-01-03 10:55:42,737 INFO SystemMonitor:155 [interfaces.py:start():190] Started gpu monitoring +2024-01-03 10:55:42,739 INFO SystemMonitor:155 [interfaces.py:start():190] Started memory monitoring +2024-01-03 10:55:42,739 INFO SystemMonitor:155 [interfaces.py:start():190] Started network monitoring +2024-01-03 10:55:42,744 DEBUG HandlerThread:155 [system_info.py:probe():196] Probing system +2024-01-03 10:55:42,747 DEBUG HandlerThread:155 [gitlib.py:_init_repo():56] git repository is invalid +2024-01-03 10:55:42,747 DEBUG HandlerThread:155 [system_info.py:probe():244] Probing system done +2024-01-03 10:55:42,747 DEBUG HandlerThread:155 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-01-03T10:55:42.744474', 'startedAt': '2024-01-03T10:55:41.493401', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': '416a3b7a3d3d', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.18, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.18, 'min': 0.0, 'max': 0.0}, {'current': 2000.18, 'min': 0.0, 'max': 0.0}, {'current': 2000.18, 'min': 0.0, 'max': 0.0}, {'current': 2000.18, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5341.223731994629}}, 'gpu': 'Tesla P100-PCIE-16GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'Tesla P100-PCIE-16GB', 'memory_total': 17071734784}], 'memory': {'total': 31.357559204101562}} +2024-01-03 10:55:42,747 INFO HandlerThread:155 [system_monitor.py:probe():224] Finished collecting system info +2024-01-03 10:55:42,747 INFO HandlerThread:155 [system_monitor.py:probe():227] Publishing system info +2024-01-03 10:55:42,748 DEBUG HandlerThread:155 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment +2024-01-03 10:55:42,749 DEBUG HandlerThread:155 [system_info.py:_save_pip():68] Saving pip packages done +2024-01-03 10:55:42,749 DEBUG HandlerThread:155 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment +2024-01-03 10:55:43,666 INFO Thread-12 :155 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files/requirements.txt +2024-01-03 10:55:43,666 INFO Thread-12 :155 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files/conda-environment.yaml +2024-01-03 10:56:33,680 INFO Thread-12 :155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files/conda-environment.yaml +2024-01-03 10:56:34,376 DEBUG HandlerThread:155 [system_info.py:_save_conda():87] Saving conda packages done +2024-01-03 10:56:34,377 INFO HandlerThread:155 [system_monitor.py:probe():229] Finished publishing system info +2024-01-03 10:56:34,384 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:56:34,385 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:56:34,385 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:56:34,385 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:56:34,385 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:56:34,385 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:56:34,385 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:56:34,385 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:56:34,385 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:56:34,385 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:56:34,385 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:56:34,386 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: stop_status +2024-01-03 10:56:34,386 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-03 10:56:34,386 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:56:34,386 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:56:34,386 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:56:34,386 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:56:34,387 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:56:34,387 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:56:34,387 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:56:34,387 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:56:34,388 DEBUG SenderThread:155 [sender.py:send_request():409] send_request: stop_status +2024-01-03 10:56:34,627 DEBUG SenderThread:155 [sender.py:send():382] send: telemetry +2024-01-03 10:56:34,627 DEBUG SenderThread:155 [sender.py:send():382] send: config +2024-01-03 10:56:34,628 DEBUG SenderThread:155 [sender.py:send():382] send: metric +2024-01-03 10:56:34,628 DEBUG SenderThread:155 [sender.py:send():382] send: telemetry +2024-01-03 10:56:34,628 DEBUG SenderThread:155 [sender.py:send():382] send: metric +2024-01-03 10:56:34,629 WARNING SenderThread:155 [sender.py:send_metric():1343] Seen metric with glob (shouldn't happen) +2024-01-03 10:56:34,629 DEBUG SenderThread:155 [sender.py:send():382] send: telemetry +2024-01-03 10:56:34,630 DEBUG SenderThread:155 [sender.py:send():382] send: files +2024-01-03 10:56:34,630 INFO SenderThread:155 [sender.py:_save_file():1392] saving file wandb-metadata.json with policy now +2024-01-03 10:56:34,681 INFO Thread-12 :155 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files/wandb-metadata.json +2024-01-03 10:56:34,681 INFO Thread-12 :155 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files/output.log +2024-01-03 10:56:35,104 INFO wandb-upload_0:155 [upload_job.py:push():131] Uploaded file /tmp/tmp95hyovy9wandb/zzsuzw8h-wandb-metadata.json +2024-01-03 10:56:36,682 INFO Thread-12 :155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files/output.log +2024-01-03 10:56:37,631 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:56:38,756 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:56:42,636 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:56:42,740 DEBUG SystemMonitor:155 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-01-03 10:56:42,876 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 10:56:43,684 INFO Thread-12 :155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files/config.yaml +2024-01-03 10:56:43,757 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:56:47,877 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:56:48,758 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:56:52,878 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:56:53,759 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:56:57,879 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:56:58,760 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:57:02,880 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:57:03,761 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:57:07,881 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:57:08,762 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:57:12,742 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 10:57:13,743 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:57:13,763 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:57:18,744 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:57:18,764 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:57:23,745 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:57:23,765 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:57:24,303 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: partial_history +2024-01-03 10:57:24,305 DEBUG SenderThread:155 [sender.py:send():382] send: metric +2024-01-03 10:57:24,305 DEBUG SenderThread:155 [sender.py:send():382] send: metric +2024-01-03 10:57:24,305 DEBUG SenderThread:155 [sender.py:send():382] send: metric +2024-01-03 10:57:24,305 DEBUG SenderThread:155 [sender.py:send():382] send: history +2024-01-03 10:57:24,305 DEBUG SenderThread:155 [sender.py:send_request():409] send_request: summary_record +2024-01-03 10:57:24,307 INFO SenderThread:155 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-03 10:57:24,699 INFO Thread-12 :155 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files/wandb-summary.json +2024-01-03 10:57:28,766 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:57:29,308 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:57:33,767 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:57:34,309 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:57:38,774 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:57:39,310 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:57:42,743 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 10:57:43,785 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:57:44,749 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:57:45,707 INFO Thread-12 :155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files/config.yaml +2024-01-03 10:57:48,786 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:57:50,049 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:57:53,787 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:57:55,050 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:57:58,787 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:58:00,051 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:58:03,788 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:58:05,052 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:58:08,790 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:58:10,053 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:58:12,743 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 10:58:13,791 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:58:15,744 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:58:18,792 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:58:20,745 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:58:23,793 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:58:25,746 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:58:28,794 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:58:30,747 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:58:33,795 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:58:34,149 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: partial_history +2024-01-03 10:58:34,150 DEBUG SenderThread:155 [sender.py:send():382] send: history +2024-01-03 10:58:34,150 DEBUG SenderThread:155 [sender.py:send_request():409] send_request: summary_record +2024-01-03 10:58:34,152 INFO SenderThread:155 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-03 10:58:34,725 INFO Thread-12 :155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files/wandb-summary.json +2024-01-03 10:58:35,726 INFO Thread-12 :155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files/output.log +2024-01-03 10:58:36,195 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:58:38,795 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:58:41,196 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:58:42,744 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 10:58:43,796 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:58:46,745 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:58:48,797 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:58:51,746 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:58:53,798 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:58:56,747 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:58:58,799 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:59:01,748 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:59:03,801 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:59:06,749 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:59:08,801 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:59:11,750 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:59:12,745 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 10:59:13,802 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:59:17,746 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:59:18,803 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:59:22,747 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:59:23,804 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:59:27,748 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:59:28,805 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:59:32,749 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:59:33,806 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:59:37,749 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:59:38,807 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:59:42,745 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 10:59:43,746 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:59:43,808 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:59:48,747 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:59:48,809 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:59:53,748 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:59:53,810 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 10:59:58,749 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 10:59:58,811 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:00:03,750 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:00:03,812 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:00:08,751 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:00:08,813 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:00:12,746 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:00:13,814 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:00:14,747 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:00:18,815 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:00:19,748 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:00:23,816 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:00:24,748 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:00:28,817 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:00:29,749 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:00:33,818 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:00:34,750 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:00:38,819 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:00:39,751 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:00:42,747 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:00:43,820 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:00:45,748 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:00:48,821 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:00:50,749 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:00:53,822 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:00:55,750 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:00:58,823 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:01:00,751 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:01:03,824 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:01:05,752 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:01:08,825 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:01:10,753 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:01:12,748 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:01:13,826 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:01:16,749 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:01:18,827 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:01:21,750 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:01:23,828 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:01:26,751 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:01:28,829 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:01:31,752 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:01:33,830 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:01:36,753 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:01:38,830 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:01:41,754 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:01:42,748 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:01:43,831 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:01:47,749 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:01:48,832 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:01:52,750 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:01:53,833 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:01:57,751 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:01:58,834 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:02:02,752 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:02:03,835 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:02:07,753 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:02:08,836 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:02:12,749 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:02:13,750 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:02:13,837 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:02:18,751 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:02:18,838 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:02:23,752 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:02:23,839 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:02:28,753 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:02:28,840 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:02:33,754 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:02:33,841 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:02:38,755 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:02:38,842 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:02:42,750 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:02:43,843 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:02:44,751 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:02:48,843 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:02:49,752 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:02:53,844 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:02:54,753 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:02:58,845 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:02:59,754 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:03:03,846 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:03:04,755 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:03:08,847 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:03:09,756 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:03:12,751 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:03:13,848 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:03:15,752 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:03:18,849 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:03:20,753 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:03:23,850 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:03:25,754 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:03:28,851 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:03:30,755 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:03:33,852 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:03:35,756 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:03:38,853 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:03:40,757 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:03:42,752 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:03:43,854 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:03:46,753 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:03:48,855 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:03:51,754 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:03:53,856 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:03:56,755 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:03:58,857 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:04:01,756 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:04:03,858 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:04:06,757 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:04:08,859 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:04:11,758 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:04:12,753 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:04:13,860 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:04:17,754 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:04:18,861 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:04:22,755 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:04:23,862 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:04:25,631 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: pause +2024-01-03 11:04:25,632 INFO HandlerThread:155 [handler.py:handle_request_pause():705] stopping system metrics thread +2024-01-03 11:04:25,632 INFO HandlerThread:155 [system_monitor.py:finish():203] Stopping system monitor +2024-01-03 11:04:25,632 DEBUG SystemMonitor:155 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-01-03 11:04:25,632 INFO HandlerThread:155 [interfaces.py:finish():202] Joined cpu monitor +2024-01-03 11:04:25,633 DEBUG SystemMonitor:155 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-01-03 11:04:25,633 INFO HandlerThread:155 [interfaces.py:finish():202] Joined disk monitor +2024-01-03 11:04:25,640 INFO HandlerThread:155 [interfaces.py:finish():202] Joined gpu monitor +2024-01-03 11:04:25,641 INFO HandlerThread:155 [interfaces.py:finish():202] Joined memory monitor +2024-01-03 11:04:25,641 INFO HandlerThread:155 [interfaces.py:finish():202] Joined network monitor +2024-01-03 11:04:25,641 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:04:28,642 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:04:28,868 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:04:33,644 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:04:33,869 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:04:38,645 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:04:38,870 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:04:43,646 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:04:43,871 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:04:48,647 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:04:48,872 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:04:53,648 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:04:53,873 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:04:58,649 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:04:58,874 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:05:03,650 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:05:03,875 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:05:08,176 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: resume +2024-01-03 11:05:08,176 INFO HandlerThread:155 [handler.py:handle_request_resume():696] starting system metrics thread +2024-01-03 11:05:08,176 INFO HandlerThread:155 [system_monitor.py:start():194] Starting system monitor +2024-01-03 11:05:08,177 INFO SystemMonitor:155 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-01-03 11:05:08,177 INFO SystemMonitor:155 [interfaces.py:start():190] Started cpu monitoring +2024-01-03 11:05:08,178 INFO SystemMonitor:155 [interfaces.py:start():190] Started disk monitoring +2024-01-03 11:05:08,179 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: pause +2024-01-03 11:05:08,179 INFO HandlerThread:155 [handler.py:handle_request_pause():705] stopping system metrics thread +2024-01-03 11:05:08,179 INFO HandlerThread:155 [system_monitor.py:finish():203] Stopping system monitor +2024-01-03 11:05:08,179 INFO SystemMonitor:155 [interfaces.py:start():190] Started gpu monitoring +2024-01-03 11:05:08,179 DEBUG SystemMonitor:155 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-01-03 11:05:08,179 DEBUG SystemMonitor:155 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-01-03 11:05:08,180 DEBUG SystemMonitor:155 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-01-03 11:05:08,182 INFO HandlerThread:155 [interfaces.py:finish():202] Joined cpu monitor +2024-01-03 11:05:08,183 INFO HandlerThread:155 [interfaces.py:finish():202] Joined disk monitor +2024-01-03 11:05:08,192 INFO HandlerThread:155 [interfaces.py:finish():202] Joined gpu monitor +2024-01-03 11:05:08,192 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:05:08,877 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:05:09,193 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:05:09,558 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: resume +2024-01-03 11:05:09,558 INFO HandlerThread:155 [handler.py:handle_request_resume():696] starting system metrics thread +2024-01-03 11:05:09,558 INFO HandlerThread:155 [system_monitor.py:start():194] Starting system monitor +2024-01-03 11:05:09,558 INFO SystemMonitor:155 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-01-03 11:05:09,559 INFO SystemMonitor:155 [interfaces.py:start():190] Started cpu monitoring +2024-01-03 11:05:09,560 INFO SystemMonitor:155 [interfaces.py:start():190] Started disk monitoring +2024-01-03 11:05:09,560 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: pause +2024-01-03 11:05:09,560 INFO HandlerThread:155 [handler.py:handle_request_pause():705] stopping system metrics thread +2024-01-03 11:05:09,560 INFO HandlerThread:155 [system_monitor.py:finish():203] Stopping system monitor +2024-01-03 11:05:09,561 INFO SystemMonitor:155 [interfaces.py:start():190] Started gpu monitoring +2024-01-03 11:05:09,561 DEBUG SystemMonitor:155 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-01-03 11:05:09,561 DEBUG SystemMonitor:155 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-01-03 11:05:09,561 DEBUG SystemMonitor:155 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-01-03 11:05:09,563 INFO HandlerThread:155 [interfaces.py:finish():202] Joined cpu monitor +2024-01-03 11:05:09,564 INFO HandlerThread:155 [interfaces.py:finish():202] Joined disk monitor +2024-01-03 11:05:09,565 INFO HandlerThread:155 [interfaces.py:finish():202] Joined gpu monitor +2024-01-03 11:05:09,565 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:05:09,779 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: resume +2024-01-03 11:05:09,779 INFO HandlerThread:155 [handler.py:handle_request_resume():696] starting system metrics thread +2024-01-03 11:05:09,779 INFO HandlerThread:155 [system_monitor.py:start():194] Starting system monitor +2024-01-03 11:05:09,779 INFO SystemMonitor:155 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-01-03 11:05:09,780 INFO SystemMonitor:155 [interfaces.py:start():190] Started cpu monitoring +2024-01-03 11:05:09,780 INFO SystemMonitor:155 [interfaces.py:start():190] Started disk monitoring +2024-01-03 11:05:09,781 INFO SystemMonitor:155 [interfaces.py:start():190] Started gpu monitoring +2024-01-03 11:05:09,783 INFO SystemMonitor:155 [interfaces.py:start():190] Started memory monitoring +2024-01-03 11:05:09,783 INFO SystemMonitor:155 [interfaces.py:start():190] Started network monitoring +2024-01-03 11:05:09,785 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: pause +2024-01-03 11:05:09,785 INFO HandlerThread:155 [handler.py:handle_request_pause():705] stopping system metrics thread +2024-01-03 11:05:09,785 INFO HandlerThread:155 [system_monitor.py:finish():203] Stopping system monitor +2024-01-03 11:05:09,786 INFO HandlerThread:155 [interfaces.py:finish():202] Joined cpu monitor +2024-01-03 11:05:09,786 DEBUG SystemMonitor:155 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-01-03 11:05:09,787 DEBUG SystemMonitor:155 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-01-03 11:05:09,787 DEBUG SystemMonitor:155 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-01-03 11:05:09,789 INFO HandlerThread:155 [interfaces.py:finish():202] Joined disk monitor +2024-01-03 11:05:09,796 INFO HandlerThread:155 [interfaces.py:finish():202] Joined gpu monitor +2024-01-03 11:05:09,796 INFO HandlerThread:155 [interfaces.py:finish():202] Joined memory monitor +2024-01-03 11:05:09,796 INFO HandlerThread:155 [interfaces.py:finish():202] Joined network monitor +2024-01-03 11:05:09,797 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:05:11,279 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: resume +2024-01-03 11:05:11,279 INFO HandlerThread:155 [handler.py:handle_request_resume():696] starting system metrics thread +2024-01-03 11:05:11,279 INFO HandlerThread:155 [system_monitor.py:start():194] Starting system monitor +2024-01-03 11:05:11,279 INFO SystemMonitor:155 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-01-03 11:05:11,279 INFO SystemMonitor:155 [interfaces.py:start():190] Started cpu monitoring +2024-01-03 11:05:11,280 INFO SystemMonitor:155 [interfaces.py:start():190] Started disk monitoring +2024-01-03 11:05:11,281 INFO SystemMonitor:155 [interfaces.py:start():190] Started gpu monitoring +2024-01-03 11:05:11,282 INFO SystemMonitor:155 [interfaces.py:start():190] Started memory monitoring +2024-01-03 11:05:11,283 INFO SystemMonitor:155 [interfaces.py:start():190] Started network monitoring +2024-01-03 11:05:11,290 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: pause +2024-01-03 11:05:11,290 INFO HandlerThread:155 [handler.py:handle_request_pause():705] stopping system metrics thread +2024-01-03 11:05:11,290 INFO HandlerThread:155 [system_monitor.py:finish():203] Stopping system monitor +2024-01-03 11:05:11,291 DEBUG SystemMonitor:155 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-01-03 11:05:11,291 DEBUG SystemMonitor:155 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-01-03 11:05:11,291 DEBUG SystemMonitor:155 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-01-03 11:05:11,291 INFO HandlerThread:155 [interfaces.py:finish():202] Joined cpu monitor +2024-01-03 11:05:11,292 INFO HandlerThread:155 [interfaces.py:finish():202] Joined disk monitor +2024-01-03 11:05:11,297 INFO HandlerThread:155 [interfaces.py:finish():202] Joined gpu monitor +2024-01-03 11:05:11,298 INFO HandlerThread:155 [interfaces.py:finish():202] Joined memory monitor +2024-01-03 11:05:11,298 INFO HandlerThread:155 [interfaces.py:finish():202] Joined network monitor +2024-01-03 11:05:11,298 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:05:12,068 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: resume +2024-01-03 11:05:12,068 INFO HandlerThread:155 [handler.py:handle_request_resume():696] starting system metrics thread +2024-01-03 11:05:12,068 INFO HandlerThread:155 [system_monitor.py:start():194] Starting system monitor +2024-01-03 11:05:12,068 INFO SystemMonitor:155 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-01-03 11:05:12,069 INFO SystemMonitor:155 [interfaces.py:start():190] Started cpu monitoring +2024-01-03 11:05:12,069 INFO SystemMonitor:155 [interfaces.py:start():190] Started disk monitoring +2024-01-03 11:05:12,070 INFO SystemMonitor:155 [interfaces.py:start():190] Started gpu monitoring +2024-01-03 11:05:12,071 INFO SystemMonitor:155 [interfaces.py:start():190] Started memory monitoring +2024-01-03 11:05:12,072 INFO SystemMonitor:155 [interfaces.py:start():190] Started network monitoring +2024-01-03 11:05:12,543 DEBUG SenderThread:155 [sender.py:send():382] send: config +2024-01-03 11:05:12,545 DEBUG SenderThread:155 [sender.py:send():382] send: metric +2024-01-03 11:05:12,545 DEBUG SenderThread:155 [sender.py:send():382] send: metric +2024-01-03 11:05:12,545 WARNING SenderThread:155 [sender.py:send_metric():1343] Seen metric with glob (shouldn't happen) +2024-01-03 11:05:13,878 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:05:14,546 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:05:18,879 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:05:19,547 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:05:23,880 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:05:24,548 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:05:28,881 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:05:29,554 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:05:29,886 INFO Thread-12 :155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files/config.yaml +2024-01-03 11:05:33,882 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:05:34,806 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:05:38,883 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:05:39,807 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:05:43,884 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:05:44,808 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:05:48,884 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:05:49,809 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:05:53,886 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:05:54,810 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:05:58,886 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:05:59,811 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:06:03,888 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:06:04,812 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:06:08,889 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:06:09,813 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:06:12,072 DEBUG SystemMonitor:155 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-01-03 11:06:12,073 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:06:13,890 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:06:15,074 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:06:18,906 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:06:20,075 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:06:22,408 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: partial_history +2024-01-03 11:06:22,409 DEBUG SenderThread:155 [sender.py:send():382] send: history +2024-01-03 11:06:22,409 DEBUG SenderThread:155 [sender.py:send_request():409] send_request: summary_record +2024-01-03 11:06:22,410 INFO SenderThread:155 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-03 11:06:22,904 INFO Thread-12 :155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files/wandb-summary.json +2024-01-03 11:06:23,907 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:06:25,411 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:06:28,911 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:06:30,412 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:06:33,911 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:06:35,413 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:06:38,912 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:06:40,414 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:06:42,074 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:06:43,913 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:06:46,075 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:06:48,914 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:06:51,077 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:06:53,915 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:06:56,078 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:06:58,916 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:07:01,079 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:07:03,917 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:07:06,080 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:07:08,918 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:07:11,081 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:07:12,075 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:07:13,919 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:07:17,076 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:07:18,920 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:07:22,077 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:07:23,921 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:07:27,078 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:07:28,922 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:07:32,079 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:07:32,243 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: partial_history +2024-01-03 11:07:32,244 DEBUG SenderThread:155 [sender.py:send():382] send: history +2024-01-03 11:07:32,245 DEBUG SenderThread:155 [sender.py:send_request():409] send_request: summary_record +2024-01-03 11:07:32,245 INFO SenderThread:155 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-03 11:07:32,929 INFO Thread-12 :155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files/wandb-summary.json +2024-01-03 11:07:33,923 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:07:37,246 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:07:38,924 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:07:42,076 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:07:43,077 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:07:43,925 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:07:48,078 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:07:48,926 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:07:53,079 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:07:53,927 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:07:58,079 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:07:58,928 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:08:03,080 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:08:03,929 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:08:08,081 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:08:08,930 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:08:12,076 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:08:13,931 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:08:14,077 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:08:18,932 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:08:19,078 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:08:23,933 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:08:24,079 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:08:28,935 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:08:29,080 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:08:33,936 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:08:34,081 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:08:38,937 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:08:39,082 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:08:42,077 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:08:43,938 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:08:45,078 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:08:48,939 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:08:50,080 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:08:53,941 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:08:55,081 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:08:58,942 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:09:00,082 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:09:03,943 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:09:05,083 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:09:08,944 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:09:10,084 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:09:12,078 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:09:13,945 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:09:16,079 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:09:18,946 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:09:21,080 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:09:23,947 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:09:26,081 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:09:28,948 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:09:31,082 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:09:33,950 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:09:36,083 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:09:38,950 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:09:41,084 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:09:42,079 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:09:43,952 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:09:47,080 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:09:48,953 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:09:52,081 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:09:53,954 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:09:57,082 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:09:58,955 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:10:02,083 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:10:03,956 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:10:07,084 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:10:08,957 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:10:12,080 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:10:13,080 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:10:13,958 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:10:18,081 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:10:18,959 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:10:23,082 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:10:23,960 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:10:28,083 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:10:28,961 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:10:33,084 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:10:33,963 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:10:38,085 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:10:38,964 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:10:42,084 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:10:43,965 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:10:44,085 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:10:48,966 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:10:49,086 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:10:53,967 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:10:54,087 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:10:58,968 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:10:59,088 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:11:03,969 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:11:04,089 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:11:08,970 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:11:09,089 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:11:12,081 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:11:13,971 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:11:15,082 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:11:18,972 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:11:20,083 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:11:23,973 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:11:25,084 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:11:28,975 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:11:30,085 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:11:33,976 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:11:35,085 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:11:38,977 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:11:40,086 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:11:42,082 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:11:43,978 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:11:46,084 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:11:48,979 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:11:51,085 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:11:53,980 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:11:56,086 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:11:58,982 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:12:01,087 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:12:03,983 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:12:06,087 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:12:08,984 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:12:11,088 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:12:12,083 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:12:13,985 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:12:17,084 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:12:18,986 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:12:22,085 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:12:23,987 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:12:27,086 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:12:28,988 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:12:32,087 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:12:33,989 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:12:37,088 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:12:38,990 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:12:42,084 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:12:43,084 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:12:43,991 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:12:48,085 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:12:48,992 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:12:53,087 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:12:53,993 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:12:58,087 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:12:58,994 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:13:03,088 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:13:03,995 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:13:08,089 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:13:08,996 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:13:12,084 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:13:13,997 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:13:14,086 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:13:18,998 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:13:19,086 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:13:23,999 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:13:24,087 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:13:29,001 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:13:29,088 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:13:34,002 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:13:34,089 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:13:39,003 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:13:39,090 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:13:42,085 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:13:44,004 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:13:45,086 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:13:49,005 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:13:50,087 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:13:54,006 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:13:55,088 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:13:59,007 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:14:00,089 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:14:04,008 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:14:05,090 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:14:09,009 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:14:10,091 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:14:12,086 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:14:14,010 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:14:16,087 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:14:19,011 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:14:21,088 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:14:24,012 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:14:26,089 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:14:29,013 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:14:31,090 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:14:34,015 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:14:36,092 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:14:39,016 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:14:41,093 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:14:42,087 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:14:44,017 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:14:47,088 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:14:49,018 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:14:52,089 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:14:54,019 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:14:57,090 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:14:59,020 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:15:02,091 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:15:04,021 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:15:07,092 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:15:09,022 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:15:12,088 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:15:13,088 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:15:14,023 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:15:18,090 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:15:19,024 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:15:23,091 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:15:24,025 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:15:28,092 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:15:29,026 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:15:33,093 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:15:34,027 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:15:38,094 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:15:39,028 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:15:42,088 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:15:44,029 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:15:44,089 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:15:49,030 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:15:49,090 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:15:54,032 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:15:54,091 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:15:59,033 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:15:59,092 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:16:04,034 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:16:04,093 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:16:08,186 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: partial_history +2024-01-03 11:16:08,189 DEBUG SenderThread:155 [sender.py:send():382] send: metric +2024-01-03 11:16:08,189 DEBUG SenderThread:155 [sender.py:send():382] send: metric +2024-01-03 11:16:08,189 DEBUG SenderThread:155 [sender.py:send():382] send: metric +2024-01-03 11:16:08,189 DEBUG SenderThread:155 [sender.py:send():382] send: metric +2024-01-03 11:16:08,189 DEBUG SenderThread:155 [sender.py:send():382] send: metric +2024-01-03 11:16:08,189 DEBUG SenderThread:155 [sender.py:send():382] send: metric +2024-01-03 11:16:08,190 DEBUG SenderThread:155 [sender.py:send():382] send: metric +2024-01-03 11:16:08,190 DEBUG SenderThread:155 [sender.py:send():382] send: history +2024-01-03 11:16:08,190 DEBUG SenderThread:155 [sender.py:send_request():409] send_request: summary_record +2024-01-03 11:16:08,191 INFO SenderThread:155 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-03 11:16:09,035 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:16:09,140 INFO Thread-12 :155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files/wandb-summary.json +2024-01-03 11:16:09,191 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:16:12,089 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:16:14,048 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:16:15,090 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:16:19,050 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:16:20,098 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:16:22,145 INFO Thread-12 :155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files/config.yaml +2024-01-03 11:16:24,051 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:16:26,420 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:16:29,052 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:16:31,420 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:16:34,053 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:16:36,421 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:16:39,054 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:16:41,423 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:16:42,090 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:16:44,055 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:16:47,091 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:16:49,056 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:16:52,092 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:16:54,063 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:16:57,093 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:16:59,066 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:17:02,094 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:17:04,067 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:17:07,095 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:17:09,068 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:17:12,091 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:17:13,092 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:17:14,069 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:17:18,093 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:17:19,070 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:17:21,520 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: partial_history +2024-01-03 11:17:21,521 DEBUG SenderThread:155 [sender.py:send():382] send: history +2024-01-03 11:17:21,521 DEBUG SenderThread:155 [sender.py:send_request():409] send_request: summary_record +2024-01-03 11:17:21,522 INFO SenderThread:155 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-03 11:17:22,170 INFO Thread-12 :155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files/wandb-summary.json +2024-01-03 11:17:23,522 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:17:24,071 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:17:28,523 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:17:29,072 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:17:33,524 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:17:34,073 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:17:38,525 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:17:39,074 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:17:42,092 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:17:44,075 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:17:44,093 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:17:49,076 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:17:49,094 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:17:54,077 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:17:54,095 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:17:59,078 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:17:59,095 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:18:04,079 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:18:04,097 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:18:09,080 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:18:09,097 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:18:12,093 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:18:14,081 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:18:15,094 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:18:19,082 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:18:20,094 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:18:24,084 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:18:25,095 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:18:29,085 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:18:30,096 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:18:31,385 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: partial_history +2024-01-03 11:18:31,386 DEBUG SenderThread:155 [sender.py:send():382] send: history +2024-01-03 11:18:31,387 DEBUG SenderThread:155 [sender.py:send_request():409] send_request: summary_record +2024-01-03 11:18:31,387 INFO SenderThread:155 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-03 11:18:32,198 INFO Thread-12 :155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files/wandb-summary.json +2024-01-03 11:18:34,086 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:18:35,388 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:18:39,087 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:18:40,390 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:18:42,093 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:18:44,089 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:18:46,095 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:18:49,090 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:18:51,095 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:18:54,091 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:18:56,096 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:18:59,092 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:19:01,097 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:19:04,093 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:19:06,098 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:19:09,094 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:19:11,099 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:19:12,094 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:19:14,095 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:19:17,096 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:19:19,096 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:19:22,097 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:19:24,097 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:19:27,098 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:19:29,098 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:19:32,099 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:19:34,100 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:19:37,101 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:19:39,101 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:19:42,095 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:19:43,096 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:19:44,102 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:19:48,097 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:19:49,103 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:19:53,098 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:19:54,104 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:19:58,099 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:19:59,105 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:20:03,100 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:20:04,106 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:20:08,101 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:20:09,107 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:20:12,096 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:20:14,097 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:20:14,108 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:20:19,098 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:20:19,109 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:20:24,098 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:20:24,110 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:20:29,099 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:20:29,111 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:20:34,100 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:20:34,112 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:20:39,101 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:20:39,113 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:20:42,097 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:20:44,114 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:20:45,098 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:20:49,115 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:20:50,099 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:20:54,116 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:20:55,100 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:20:59,118 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:21:00,101 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:21:04,119 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:21:05,102 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:21:09,120 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:21:10,103 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:21:12,097 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:21:14,121 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:21:16,098 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:21:19,122 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:21:21,099 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:21:24,123 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:21:26,100 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:21:29,124 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:21:31,101 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:21:34,125 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:21:36,102 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:21:39,126 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:21:41,103 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:21:42,098 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:21:44,127 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:21:47,099 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:21:49,128 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:21:52,100 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:21:54,130 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:21:57,101 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:21:57,598 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: partial_history +2024-01-03 11:21:57,599 DEBUG SenderThread:155 [sender.py:send():382] send: history +2024-01-03 11:21:57,600 DEBUG SenderThread:155 [sender.py:send_request():409] send_request: summary_record +2024-01-03 11:21:57,600 INFO SenderThread:155 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-03 11:21:58,280 INFO Thread-12 :155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files/wandb-summary.json +2024-01-03 11:21:59,167 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:22:02,601 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:22:04,354 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:22:07,602 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:22:09,355 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:22:12,099 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:22:13,100 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:22:14,357 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:22:18,101 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:22:19,358 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:22:23,102 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:22:24,359 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:22:28,103 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:22:29,359 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:22:33,104 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:22:34,360 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:22:38,105 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:22:39,362 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:22:42,100 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:22:44,100 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:22:44,362 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:22:49,101 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:22:49,364 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:22:54,102 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:22:54,365 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:22:59,104 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:22:59,366 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:23:04,104 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:23:04,383 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:23:09,105 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:23:09,384 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:23:10,857 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: partial_history +2024-01-03 11:23:10,858 DEBUG SenderThread:155 [sender.py:send():382] send: history +2024-01-03 11:23:10,859 DEBUG SenderThread:155 [sender.py:send_request():409] send_request: summary_record +2024-01-03 11:23:10,859 INFO SenderThread:155 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-03 11:23:11,310 INFO Thread-12 :155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files/wandb-summary.json +2024-01-03 11:23:12,100 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:23:14,385 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:23:15,101 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:23:19,386 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:23:20,102 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:23:24,387 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:23:25,103 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:23:29,389 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:23:30,105 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:23:34,390 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:23:35,105 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:23:39,391 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:23:40,106 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:23:42,101 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:23:44,392 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:23:46,102 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:23:49,393 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:23:51,103 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:23:54,394 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:23:56,104 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:23:59,395 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:24:01,105 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:24:04,396 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:24:06,107 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:24:09,398 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:24:11,108 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:24:12,102 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:24:14,399 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:24:17,103 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:24:19,400 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:24:20,693 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: partial_history +2024-01-03 11:24:20,694 DEBUG SenderThread:155 [sender.py:send():382] send: history +2024-01-03 11:24:20,695 DEBUG SenderThread:155 [sender.py:send_request():409] send_request: summary_record +2024-01-03 11:24:20,695 INFO SenderThread:155 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-03 11:24:21,338 INFO Thread-12 :155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files/wandb-summary.json +2024-01-03 11:24:22,696 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:24:24,401 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:24:27,697 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:24:29,402 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:24:32,698 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:24:34,402 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:24:37,699 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:24:39,404 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:24:42,103 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:24:43,104 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:24:44,405 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:24:48,105 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:24:49,406 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:24:53,106 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:24:54,407 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:24:58,107 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:24:59,408 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:25:03,108 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:25:04,409 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:25:08,109 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:25:09,410 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:25:12,104 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:25:14,105 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:25:14,411 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:25:19,106 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:25:19,411 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:25:24,107 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:25:24,413 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:25:29,108 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:25:29,414 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:25:34,109 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:25:34,415 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:25:39,110 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:25:39,416 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:25:42,105 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:25:44,417 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:25:45,106 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:25:49,418 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:25:50,107 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:25:54,419 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:25:55,108 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:25:59,420 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:26:00,109 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:26:04,421 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:26:05,110 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:26:09,422 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:26:10,110 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:26:12,105 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:26:14,423 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:26:16,106 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:26:19,424 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:26:21,107 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:26:24,425 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:26:26,108 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:26:29,426 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:26:31,109 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:26:34,427 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:26:36,110 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:26:39,428 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:26:41,111 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:26:42,106 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:26:44,430 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:26:47,107 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:26:49,431 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:26:52,108 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:26:54,432 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:26:57,109 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:26:59,433 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:27:02,110 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:27:04,433 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:27:07,111 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:27:09,434 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:27:12,107 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:27:13,108 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:27:14,435 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:27:18,109 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:27:19,436 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:27:23,110 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:27:24,437 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:27:28,111 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:27:29,438 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:27:33,112 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:27:34,439 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:27:36,162 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: partial_history +2024-01-03 11:27:36,164 DEBUG SenderThread:155 [sender.py:send():382] send: history +2024-01-03 11:27:36,164 DEBUG SenderThread:155 [sender.py:send_request():409] send_request: summary_record +2024-01-03 11:27:36,165 INFO SenderThread:155 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-03 11:27:36,413 INFO Thread-12 :155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files/wandb-summary.json +2024-01-03 11:27:38,166 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:27:39,534 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:27:42,108 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:27:44,108 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:27:44,535 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:27:49,109 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:27:49,536 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:27:53,761 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: partial_history +2024-01-03 11:27:53,764 DEBUG SenderThread:155 [sender.py:send():382] send: metric +2024-01-03 11:27:53,764 DEBUG SenderThread:155 [sender.py:send():382] send: metric +2024-01-03 11:27:53,764 DEBUG SenderThread:155 [sender.py:send():382] send: metric +2024-01-03 11:27:53,764 DEBUG SenderThread:155 [sender.py:send():382] send: metric +2024-01-03 11:27:53,764 DEBUG SenderThread:155 [sender.py:send():382] send: metric +2024-01-03 11:27:53,764 DEBUG SenderThread:155 [sender.py:send():382] send: history +2024-01-03 11:27:53,765 DEBUG SenderThread:155 [sender.py:send_request():409] send_request: summary_record +2024-01-03 11:27:53,765 INFO SenderThread:155 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-03 11:27:53,766 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: pause +2024-01-03 11:27:53,766 INFO HandlerThread:155 [handler.py:handle_request_pause():705] stopping system metrics thread +2024-01-03 11:27:53,766 INFO HandlerThread:155 [system_monitor.py:finish():203] Stopping system monitor +2024-01-03 11:27:53,767 DEBUG SystemMonitor:155 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-01-03 11:27:53,767 DEBUG SystemMonitor:155 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-01-03 11:27:53,768 INFO HandlerThread:155 [interfaces.py:finish():202] Joined cpu monitor +2024-01-03 11:27:53,768 INFO HandlerThread:155 [interfaces.py:finish():202] Joined disk monitor +2024-01-03 11:27:53,774 INFO HandlerThread:155 [interfaces.py:finish():202] Joined gpu monitor +2024-01-03 11:27:53,774 INFO HandlerThread:155 [interfaces.py:finish():202] Joined memory monitor +2024-01-03 11:27:53,774 INFO HandlerThread:155 [interfaces.py:finish():202] Joined network monitor +2024-01-03 11:27:53,775 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:27:54,419 INFO Thread-12 :155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files/wandb-summary.json +2024-01-03 11:27:54,537 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:27:54,776 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:27:59,538 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:27:59,777 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:28:04,540 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:28:04,778 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:28:09,541 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:28:09,779 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:28:14,683 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:28:14,785 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:28:16,427 INFO Thread-12 :155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240103_105541-kk0m5fl8/files/config.yaml +2024-01-03 11:28:19,684 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:28:21,116 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:28:23,469 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: resume +2024-01-03 11:28:23,469 INFO HandlerThread:155 [handler.py:handle_request_resume():696] starting system metrics thread +2024-01-03 11:28:23,469 INFO HandlerThread:155 [system_monitor.py:start():194] Starting system monitor +2024-01-03 11:28:23,469 INFO SystemMonitor:155 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-01-03 11:28:23,470 INFO SystemMonitor:155 [interfaces.py:start():190] Started cpu monitoring +2024-01-03 11:28:23,470 INFO SystemMonitor:155 [interfaces.py:start():190] Started disk monitoring +2024-01-03 11:28:23,471 INFO SystemMonitor:155 [interfaces.py:start():190] Started gpu monitoring +2024-01-03 11:28:23,473 INFO SystemMonitor:155 [interfaces.py:start():190] Started memory monitoring +2024-01-03 11:28:23,475 INFO SystemMonitor:155 [interfaces.py:start():190] Started network monitoring +2024-01-03 11:28:24,109 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: pause +2024-01-03 11:28:24,110 INFO HandlerThread:155 [handler.py:handle_request_pause():705] stopping system metrics thread +2024-01-03 11:28:24,110 INFO HandlerThread:155 [system_monitor.py:finish():203] Stopping system monitor +2024-01-03 11:28:24,111 DEBUG SystemMonitor:155 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-01-03 11:28:24,111 DEBUG SystemMonitor:155 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-01-03 11:28:24,111 DEBUG SystemMonitor:155 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-01-03 11:28:24,112 INFO HandlerThread:155 [interfaces.py:finish():202] Joined cpu monitor +2024-01-03 11:28:24,112 INFO HandlerThread:155 [interfaces.py:finish():202] Joined disk monitor +2024-01-03 11:28:24,120 INFO HandlerThread:155 [interfaces.py:finish():202] Joined gpu monitor +2024-01-03 11:28:24,120 INFO HandlerThread:155 [interfaces.py:finish():202] Joined memory monitor +2024-01-03 11:28:24,120 INFO HandlerThread:155 [interfaces.py:finish():202] Joined network monitor +2024-01-03 11:28:24,120 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:28:24,689 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:28:26,121 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:28:29,690 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:28:31,122 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:28:34,691 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:28:36,123 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:28:39,692 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:28:41,124 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:28:44,693 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:28:46,125 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:28:49,694 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:28:51,126 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:28:54,695 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:28:56,127 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:28:59,696 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:29:01,128 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:29:04,697 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:29:06,129 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:29:06,753 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: resume +2024-01-03 11:29:06,753 INFO HandlerThread:155 [handler.py:handle_request_resume():696] starting system metrics thread +2024-01-03 11:29:06,753 INFO HandlerThread:155 [system_monitor.py:start():194] Starting system monitor +2024-01-03 11:29:06,753 INFO SystemMonitor:155 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-01-03 11:29:06,754 INFO SystemMonitor:155 [interfaces.py:start():190] Started cpu monitoring +2024-01-03 11:29:06,755 INFO SystemMonitor:155 [interfaces.py:start():190] Started disk monitoring +2024-01-03 11:29:06,755 INFO SystemMonitor:155 [interfaces.py:start():190] Started gpu monitoring +2024-01-03 11:29:06,756 INFO SystemMonitor:155 [interfaces.py:start():190] Started memory monitoring +2024-01-03 11:29:06,758 INFO SystemMonitor:155 [interfaces.py:start():190] Started network monitoring +2024-01-03 11:29:06,780 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: pause +2024-01-03 11:29:06,780 INFO HandlerThread:155 [handler.py:handle_request_pause():705] stopping system metrics thread +2024-01-03 11:29:06,780 INFO HandlerThread:155 [system_monitor.py:finish():203] Stopping system monitor +2024-01-03 11:29:06,780 DEBUG SystemMonitor:155 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-01-03 11:29:06,780 DEBUG SystemMonitor:155 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-01-03 11:29:06,781 DEBUG SystemMonitor:155 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-01-03 11:29:06,782 INFO HandlerThread:155 [interfaces.py:finish():202] Joined cpu monitor +2024-01-03 11:29:06,782 INFO HandlerThread:155 [interfaces.py:finish():202] Joined disk monitor +2024-01-03 11:29:06,790 INFO HandlerThread:155 [interfaces.py:finish():202] Joined gpu monitor +2024-01-03 11:29:06,790 INFO HandlerThread:155 [interfaces.py:finish():202] Joined memory monitor +2024-01-03 11:29:06,790 INFO HandlerThread:155 [interfaces.py:finish():202] Joined network monitor +2024-01-03 11:29:06,791 DEBUG SenderThread:155 [sender.py:send():382] send: stats +2024-01-03 11:29:09,698 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:29:11,792 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:29:14,700 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:29:16,793 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:29:19,701 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:29:21,794 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:29:24,702 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:29:26,795 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:29:29,703 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:29:31,796 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:29:34,704 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:29:36,797 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:29:39,705 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:29:41,798 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:29:44,706 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:29:46,799 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:29:49,708 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:29:51,800 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:29:54,709 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:29:56,570 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: resume +2024-01-03 11:29:56,570 INFO HandlerThread:155 [handler.py:handle_request_resume():696] starting system metrics thread +2024-01-03 11:29:56,570 INFO HandlerThread:155 [system_monitor.py:start():194] Starting system monitor +2024-01-03 11:29:56,571 INFO SystemMonitor:155 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-01-03 11:29:56,571 INFO SystemMonitor:155 [interfaces.py:start():190] Started cpu monitoring +2024-01-03 11:29:56,572 INFO SystemMonitor:155 [interfaces.py:start():190] Started disk monitoring +2024-01-03 11:29:56,574 INFO SystemMonitor:155 [interfaces.py:start():190] Started gpu monitoring +2024-01-03 11:29:56,574 INFO SystemMonitor:155 [interfaces.py:start():190] Started memory monitoring +2024-01-03 11:29:56,575 INFO SystemMonitor:155 [interfaces.py:start():190] Started network monitoring +2024-01-03 11:29:56,801 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:30:00,186 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:30:01,802 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:30:05,187 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:30:06,803 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:30:10,188 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:30:11,804 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report +2024-01-03 11:30:15,189 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: keepalive +2024-01-03 11:30:16,805 DEBUG HandlerThread:155 [handler.py:handle_request():146] handle_request: status_report