diff --git "a/wandb/run-20240314_140503-88c6vfns/logs/debug-internal.log" "b/wandb/run-20240314_140503-88c6vfns/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20240314_140503-88c6vfns/logs/debug-internal.log" @@ -0,0 +1,2138 @@ +2024-03-14 14:05:03,189 INFO StreamThr :195 [internal.py:wandb_internal():86] W&B internal server running at pid: 195, started at: 2024-03-14 14:05:03.188803 +2024-03-14 14:05:03,191 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status +2024-03-14 14:05:03,581 INFO WriterThread:195 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240314_140503-88c6vfns/run-88c6vfns.wandb +2024-03-14 14:05:03,582 DEBUG SenderThread:195 [sender.py:send():382] send: header +2024-03-14 14:05:03,585 DEBUG SenderThread:195 [sender.py:send():382] send: run +2024-03-14 14:05:03,745 INFO SenderThread:195 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files +2024-03-14 14:05:03,745 INFO SenderThread:195 [sender.py:_start_run_threads():1136] run started: 88c6vfns with start time 1710425103.190049 +2024-03-14 14:05:03,752 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: check_version +2024-03-14 14:05:03,753 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: check_version +2024-03-14 14:05:03,843 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: run_start +2024-03-14 14:05:03,855 DEBUG HandlerThread:195 [system_info.py:__init__():27] System info init +2024-03-14 14:05:03,855 DEBUG HandlerThread:195 [system_info.py:__init__():42] System info init done +2024-03-14 14:05:03,855 INFO HandlerThread:195 [system_monitor.py:start():194] Starting system monitor +2024-03-14 14:05:03,856 INFO SystemMonitor:195 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-14 14:05:03,856 INFO HandlerThread:195 [system_monitor.py:probe():214] Collecting system info +2024-03-14 14:05:03,857 INFO SystemMonitor:195 [interfaces.py:start():190] Started cpu monitoring +2024-03-14 14:05:03,857 INFO SystemMonitor:195 [interfaces.py:start():190] Started disk monitoring +2024-03-14 14:05:03,859 INFO SystemMonitor:195 [interfaces.py:start():190] Started gpu monitoring +2024-03-14 14:05:03,859 INFO SystemMonitor:195 [interfaces.py:start():190] Started memory monitoring +2024-03-14 14:05:03,860 INFO SystemMonitor:195 [interfaces.py:start():190] Started network monitoring +2024-03-14 14:05:03,870 DEBUG HandlerThread:195 [system_info.py:probe():151] Probing system +2024-03-14 14:05:03,872 DEBUG HandlerThread:195 [gitlib.py:_init_repo():56] git repository is invalid +2024-03-14 14:05:03,872 DEBUG HandlerThread:195 [system_info.py:probe():199] Probing system done +2024-03-14 14:05:03,872 DEBUG HandlerThread:195 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-03-14T14:05:03.870282', 'startedAt': '2024-03-14T14:05:03.182323', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': '0a91acd6c6df', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.138, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.138, 'min': 0.0, 'max': 0.0}, {'current': 2000.138, 'min': 0.0, 'max': 0.0}, {'current': 2000.138, 'min': 0.0, 'max': 0.0}, {'current': 2000.138, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5523.308406829834}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.35755157470703}} +2024-03-14 14:05:03,873 INFO HandlerThread:195 [system_monitor.py:probe():224] Finished collecting system info +2024-03-14 14:05:03,873 INFO HandlerThread:195 [system_monitor.py:probe():227] Publishing system info +2024-03-14 14:05:03,873 DEBUG HandlerThread:195 [system_info.py:_save_conda():208] Saving list of conda packages installed into the current environment +2024-03-14 14:05:04,747 INFO Thread-12 :195 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/conda-environment.yaml +2024-03-14 14:05:59,764 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/conda-environment.yaml +2024-03-14 14:05:59,923 DEBUG HandlerThread:195 [system_info.py:_save_conda():220] Saving conda packages done +2024-03-14 14:05:59,924 INFO HandlerThread:195 [system_monitor.py:probe():229] Finished publishing system info +2024-03-14 14:05:59,934 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:05:59,934 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:05:59,935 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:05:59,935 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:05:59,935 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:05:59,935 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:05:59,935 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:05:59,935 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:05:59,935 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:05:59,935 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:05:59,935 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:05:59,935 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: python_packages +2024-03-14 14:05:59,936 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: stop_status +2024-03-14 14:05:59,936 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-14 14:05:59,936 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:05:59,936 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:05:59,936 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:05:59,936 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:05:59,936 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:05:59,936 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:05:59,936 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:05:59,936 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:05:59,936 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:05:59,936 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:05:59,937 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: python_packages +2024-03-14 14:05:59,941 DEBUG SenderThread:195 [sender.py:send():382] send: telemetry +2024-03-14 14:05:59,941 DEBUG SenderThread:195 [sender.py:send():382] send: config +2024-03-14 14:05:59,943 DEBUG SenderThread:195 [sender.py:send():382] send: metric +2024-03-14 14:05:59,943 DEBUG SenderThread:195 [sender.py:send():382] send: telemetry +2024-03-14 14:05:59,943 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: stop_status +2024-03-14 14:06:00,033 DEBUG SenderThread:195 [sender.py:send():382] send: metric +2024-03-14 14:06:00,033 WARNING SenderThread:195 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) +2024-03-14 14:06:00,034 DEBUG SenderThread:195 [sender.py:send():382] send: telemetry +2024-03-14 14:06:00,034 DEBUG SenderThread:195 [sender.py:send():382] send: files +2024-03-14 14:06:00,035 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-03-14 14:06:00,253 INFO wandb-upload_0:195 [upload_job.py:push():131] Uploaded file /tmp/tmp6euwfyvwwandb/bcyqcg3m-wandb-metadata.json +2024-03-14 14:06:00,765 INFO Thread-12 :195 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:06:00,765 INFO Thread-12 :195 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-metadata.json +2024-03-14 14:06:00,766 INFO Thread-12 :195 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/requirements.txt +2024-03-14 14:06:02,766 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:06:03,860 DEBUG SystemMonitor:195 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-14 14:06:03,862 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:06:03,868 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:06:03,893 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:06:04,766 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/config.yaml +2024-03-14 14:06:08,894 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:06:09,004 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:06:13,896 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:06:14,004 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:06:18,897 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:06:19,005 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:06:23,898 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:06:24,006 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:06:28,899 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:06:29,007 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:06:33,863 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:06:33,900 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:06:34,864 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:06:35,764 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:06:35,767 DEBUG SenderThread:195 [sender.py:send():382] send: metric +2024-03-14 14:06:35,767 DEBUG SenderThread:195 [sender.py:send():382] send: metric +2024-03-14 14:06:35,767 DEBUG SenderThread:195 [sender.py:send():382] send: metric +2024-03-14 14:06:35,767 DEBUG SenderThread:195 [sender.py:send():382] send: metric +2024-03-14 14:06:35,767 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:06:35,767 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:06:35,768 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:06:35,777 INFO Thread-12 :195 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:06:38,901 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:06:40,769 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:06:41,723 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:06:41,726 DEBUG SenderThread:195 [sender.py:send():382] send: metric +2024-03-14 14:06:41,726 DEBUG SenderThread:195 [sender.py:send():382] send: metric +2024-03-14 14:06:41,726 DEBUG SenderThread:195 [sender.py:send():382] send: metric +2024-03-14 14:06:41,726 DEBUG SenderThread:195 [sender.py:send():382] send: metric +2024-03-14 14:06:41,726 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:06:41,727 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:06:41,727 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:06:41,779 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:06:43,903 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:06:44,781 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:06:45,959 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:06:48,904 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:06:50,960 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:06:53,905 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:06:55,961 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:06:58,906 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:07:00,962 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:07:03,864 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:07:03,907 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:07:06,871 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:07:08,790 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/config.yaml +2024-03-14 14:07:08,908 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:07:13,674 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:07:13,909 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:07:18,675 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:07:18,910 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:07:23,676 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:07:23,911 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:07:28,677 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:07:28,912 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:07:33,677 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:07:33,864 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:07:33,913 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:07:38,866 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:07:38,914 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:07:39,838 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:07:39,839 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:07:39,839 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:07:39,839 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:07:40,803 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:07:43,915 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:07:44,840 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:07:45,958 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:07:45,960 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:07:45,960 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:07:45,962 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:07:46,805 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:07:48,806 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:07:48,916 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:07:50,257 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:07:53,916 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:07:55,258 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:07:58,917 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:08:00,259 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:08:03,865 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:08:03,918 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:08:05,866 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:08:08,919 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:08:10,867 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:08:13,920 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:08:15,868 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:08:18,921 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:08:20,870 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:08:23,922 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:08:25,871 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:08:28,924 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:08:30,871 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:08:33,866 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:08:33,925 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:08:36,867 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:08:38,926 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:08:41,500 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:08:41,501 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:08:41,501 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:08:41,502 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:08:41,829 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:08:42,503 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:08:43,927 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:08:47,501 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:08:47,502 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:08:47,502 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:08:47,504 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:08:47,505 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:08:47,831 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:08:48,927 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:08:50,832 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:08:52,750 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:08:53,928 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:08:57,751 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:08:58,929 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:09:02,752 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:09:03,867 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:09:03,930 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:09:07,869 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:09:08,931 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:09:12,869 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:09:13,932 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:09:17,871 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:09:18,933 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:09:22,872 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:09:23,934 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:09:27,872 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:09:28,935 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:09:32,873 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:09:33,868 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:09:33,939 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:09:38,870 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:09:38,940 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:09:43,871 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:09:43,941 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:09:48,871 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:09:48,942 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:09:49,029 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:09:49,030 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:09:49,030 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:09:49,033 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:09:49,855 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:09:53,943 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:09:54,034 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:09:55,137 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:09:55,138 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:09:55,138 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:09:55,138 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:09:55,857 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:09:56,858 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:09:58,944 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:09:59,345 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:10:03,870 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:10:03,945 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:10:04,870 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:10:08,946 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:10:09,871 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:10:13,947 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:10:14,872 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:10:18,948 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:10:19,873 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:10:23,949 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:10:24,873 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:10:28,950 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:10:29,874 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:10:33,870 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:10:33,952 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:10:35,873 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:10:38,953 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:10:40,874 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:10:43,954 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:10:45,875 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:10:48,954 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:10:50,876 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:10:53,955 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:10:55,877 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:10:58,504 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:10:58,505 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:10:58,505 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:10:58,506 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:10:58,884 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:10:58,956 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:11:01,506 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:11:03,871 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:11:03,957 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:11:04,502 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:11:04,503 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:11:04,503 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:11:04,504 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:11:04,886 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:11:06,701 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:11:06,887 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:11:08,958 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:11:11,702 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:11:13,959 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:11:16,703 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:11:18,960 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:11:21,704 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:11:23,962 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:11:26,705 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:11:28,962 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:11:31,706 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:11:33,872 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:11:33,963 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:11:36,873 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:11:38,964 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:11:41,874 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:11:43,965 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:11:46,875 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:11:48,966 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:11:51,876 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:11:53,967 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:11:56,877 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:11:58,968 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:12:01,878 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:12:03,873 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:12:03,969 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:12:05,646 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:12:05,647 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:12:05,647 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:12:05,648 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:12:05,910 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:12:07,649 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:12:08,971 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:12:11,574 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:12:11,575 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:12:11,576 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:12:11,576 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:12:11,913 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:12:12,775 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:12:12,914 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:12:13,972 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:12:17,776 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:12:18,973 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:12:22,776 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:12:23,974 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:12:27,777 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:12:28,975 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:12:32,778 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:12:33,874 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:12:33,976 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:12:37,875 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:12:38,977 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:12:42,876 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:12:43,978 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:12:47,877 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:12:48,979 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:12:52,878 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:12:53,980 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:12:57,879 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:12:58,981 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:13:02,880 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:13:03,875 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:13:03,982 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:13:08,876 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:13:08,983 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:13:11,723 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:13:11,724 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:13:11,724 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:13:11,725 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:13:11,938 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:13:13,984 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:13:14,725 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:13:17,706 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:13:17,707 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:13:17,707 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:13:17,708 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:13:17,940 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:13:18,941 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:13:18,985 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:13:19,918 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:13:23,986 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:13:24,919 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:13:28,987 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:13:29,920 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:13:33,876 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:13:33,988 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:13:35,877 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:13:38,989 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:13:40,878 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:13:43,990 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:13:45,879 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:13:48,991 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:13:50,880 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:13:53,992 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:13:55,881 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:13:58,993 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:14:00,882 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:14:03,877 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:14:03,994 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:14:06,879 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:14:08,995 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:14:11,880 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:14:13,996 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:14:16,881 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:14:18,997 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:14:19,189 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:14:19,191 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:14:19,191 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:14:19,192 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:14:19,964 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:14:22,193 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:14:23,998 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:14:25,148 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:14:25,150 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:14:25,150 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:14:25,150 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:14:25,966 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:14:26,967 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:14:27,376 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:14:28,999 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:14:32,377 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:14:33,878 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:14:34,000 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:14:37,879 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:14:39,001 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:14:42,880 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:14:44,002 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:14:47,881 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:14:49,003 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:14:52,882 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:14:54,004 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:14:57,883 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:14:59,005 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:15:02,884 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:15:03,879 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:15:04,006 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:15:08,881 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:15:09,007 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:15:13,882 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:15:14,008 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:15:18,883 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:15:19,010 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:15:23,884 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:15:24,011 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:15:25,667 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:15:25,668 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:15:25,669 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:15:25,669 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:15:25,991 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:15:29,012 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:15:29,670 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:15:31,617 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:15:31,623 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:15:31,624 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:15:31,624 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:15:31,993 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:15:32,994 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:15:33,880 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:15:34,013 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:15:34,881 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:15:39,014 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:15:39,882 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:15:44,015 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:15:44,883 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:15:49,016 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:15:49,884 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:15:54,017 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:15:54,885 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:15:59,018 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:15:59,886 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:16:03,881 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:16:04,019 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:16:05,883 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:16:09,020 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:16:10,884 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:16:14,021 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:16:15,885 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:16:19,022 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:16:20,885 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:16:24,023 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:16:25,887 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:16:29,025 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:16:30,887 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:16:31,890 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:16:31,891 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:16:31,892 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:16:31,892 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:16:32,016 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:16:33,883 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:16:34,025 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:16:36,884 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:16:37,916 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:16:37,917 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:16:37,918 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:16:37,918 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:16:38,019 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:16:39,020 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:16:39,027 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:16:42,123 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:16:44,028 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:16:47,124 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:16:49,028 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:16:52,125 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:16:54,029 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:16:57,126 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:16:59,030 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:17:02,127 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:17:03,884 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:17:04,031 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:17:07,885 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:17:09,032 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:17:12,886 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:17:14,033 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:17:17,887 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:17:19,034 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:17:22,888 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:17:24,035 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:17:27,889 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:17:29,036 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:17:32,890 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:17:33,885 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:17:34,037 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:17:38,886 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:17:39,037 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:17:39,106 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:17:39,107 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:17:39,108 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:17:39,108 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:17:40,044 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:17:44,038 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:17:44,109 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:17:45,066 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:17:45,067 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:17:45,067 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:17:45,067 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:17:46,046 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:17:47,046 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:17:49,039 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:17:49,527 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:17:54,040 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:17:54,527 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:17:59,041 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:17:59,528 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:18:03,886 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:18:04,042 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:18:04,886 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:18:09,044 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:18:09,887 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:18:14,045 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:18:14,888 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:18:19,046 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:18:19,889 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:18:24,047 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:18:24,890 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:18:29,048 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:18:29,891 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:18:33,887 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:18:34,049 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:18:35,888 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:18:39,050 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:18:40,889 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:18:44,051 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:18:44,480 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:18:44,481 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:18:44,481 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:18:44,481 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:18:45,070 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:18:46,482 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:18:49,052 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:18:50,580 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:18:50,582 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:18:50,582 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:18:50,582 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:18:51,072 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:18:51,799 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:18:53,073 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:18:54,053 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:18:56,800 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:18:59,054 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:19:01,801 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:19:03,887 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:19:04,055 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:19:06,889 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:19:09,056 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:19:11,890 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:19:14,057 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:19:16,891 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:19:19,057 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:19:21,891 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:19:24,058 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:19:26,892 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:19:29,059 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:19:31,893 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:19:33,889 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:19:34,060 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:19:37,890 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:19:39,061 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:19:42,891 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:19:44,062 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:19:47,892 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:19:49,063 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:19:52,893 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:19:53,109 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:19:53,110 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:19:53,110 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:19:53,111 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:19:54,064 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:19:54,095 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:19:58,112 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:19:59,065 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:19:59,098 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:19:59,100 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:19:59,100 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:19:59,101 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:20:00,098 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:20:01,098 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:20:03,297 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:20:03,890 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:20:04,066 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:20:08,891 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:20:09,067 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:20:13,892 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:20:14,068 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:20:18,893 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:20:19,069 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:20:23,894 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:20:24,071 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:20:28,895 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:20:29,072 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:20:33,891 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:20:34,073 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:20:34,892 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:20:39,074 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:20:39,893 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:20:44,075 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:20:44,894 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:20:49,076 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:20:49,895 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:20:54,077 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:20:54,896 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:20:59,077 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:20:59,897 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:21:00,715 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:21:00,716 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:21:00,716 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:21:00,717 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:21:01,122 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:21:03,892 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:21:04,078 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:21:05,893 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:21:06,709 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:21:06,710 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:21:06,710 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:21:06,711 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:21:07,124 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:21:09,079 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:21:09,125 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:21:10,918 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:21:14,080 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:21:15,919 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:21:19,081 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:21:20,920 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:21:24,082 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:21:25,921 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:21:29,083 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:21:30,922 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:21:33,893 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:21:34,085 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:21:36,894 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:21:39,085 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:21:41,895 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:21:44,086 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:21:46,896 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:21:49,091 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:21:51,897 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:21:54,091 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:21:56,898 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:21:59,093 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:22:01,899 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:22:03,894 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:22:04,094 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:22:07,895 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:22:08,578 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:22:08,579 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:22:08,579 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:22:08,580 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:22:09,095 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:22:09,148 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:22:13,581 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:22:14,095 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:22:14,644 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:22:14,645 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:22:14,645 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:22:14,645 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:22:15,150 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:22:17,151 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:22:18,888 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:22:19,096 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:22:23,889 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:22:24,097 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:22:28,890 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:22:29,099 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:22:33,891 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:22:33,894 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:22:34,100 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:22:38,896 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:22:39,100 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:22:43,897 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:22:44,102 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:22:48,898 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:22:49,103 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:22:53,899 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:22:54,103 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:22:58,900 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:22:59,105 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:23:03,895 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:23:04,106 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:23:04,896 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:23:09,107 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:23:09,897 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:23:14,108 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:23:14,898 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:23:16,642 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:23:16,644 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:23:16,644 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:23:16,644 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:23:17,176 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:23:19,109 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:23:20,646 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:23:22,640 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:23:22,642 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:23:22,642 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:23:22,643 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:23:23,179 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:23:24,110 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:23:25,180 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:23:25,864 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:23:29,111 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:23:30,865 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:23:33,896 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:23:34,112 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:23:35,897 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:23:39,113 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:23:40,899 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:23:44,114 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:23:45,900 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:23:49,115 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:23:50,901 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:23:54,116 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:23:55,902 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:23:59,117 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:24:00,904 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:24:03,897 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:24:04,118 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:24:06,898 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:24:09,119 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:24:11,899 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:24:14,120 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:24:16,900 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:24:19,121 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:24:21,901 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:24:22,524 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:24:22,526 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:24:22,526 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:24:22,526 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:24:23,205 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:24:24,122 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:24:27,527 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:24:28,509 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:24:28,510 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:24:28,511 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:24:28,511 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:24:29,123 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:24:29,208 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:24:30,208 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:24:32,731 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:24:33,898 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:24:34,124 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:24:37,899 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:24:39,125 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:24:42,900 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:24:44,126 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:24:47,902 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:24:49,127 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:24:52,903 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:24:54,128 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:24:57,904 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:24:59,129 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:25:02,905 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:25:03,899 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:25:04,131 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:25:08,900 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:25:09,132 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:25:13,901 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:25:14,133 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:25:18,902 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:25:19,134 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:25:23,904 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:25:24,135 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:25:28,905 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:25:29,137 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:25:30,171 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:25:30,172 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:25:30,173 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:25:30,173 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:25:30,233 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:25:33,900 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:25:34,138 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:25:34,901 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:25:36,138 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:25:36,140 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:25:36,140 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:25:36,140 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:25:36,236 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:25:38,237 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:25:39,139 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:25:40,352 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:25:44,140 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:25:45,353 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:25:49,141 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:25:50,354 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:25:54,142 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:25:55,355 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:25:59,144 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:26:00,356 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:26:03,901 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:26:04,145 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:26:05,902 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:26:09,146 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:26:10,903 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:26:14,147 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:26:15,904 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:26:19,148 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:26:20,905 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:26:24,149 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:26:25,906 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:26:29,151 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:26:30,907 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:26:33,902 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:26:34,152 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:26:34,785 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:26:34,786 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:26:34,786 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:26:34,787 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:26:35,260 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:26:36,788 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:26:39,153 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:26:40,750 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:26:40,751 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:26:40,751 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:26:40,752 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:26:41,262 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:26:41,958 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:26:42,263 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:26:44,154 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:26:46,959 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:26:49,155 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:26:51,960 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:26:54,156 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:26:56,961 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:26:59,157 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:27:01,962 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:27:03,903 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:27:04,158 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:27:07,904 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:27:09,159 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:27:12,905 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:27:14,160 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:27:17,906 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:27:19,161 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:27:22,907 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:27:24,162 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:27:27,908 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:27:29,163 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:27:32,909 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:27:33,904 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:27:34,165 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:27:38,905 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:27:39,166 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:27:43,409 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:27:43,416 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:27:43,416 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:27:43,417 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:27:44,167 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:27:44,287 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:27:44,418 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:27:49,168 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:27:49,388 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:27:49,389 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:27:49,389 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:27:49,390 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:27:49,595 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:27:50,290 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:27:52,291 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:27:54,169 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:27:54,597 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:27:59,170 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:27:59,597 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:28:03,905 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:28:04,171 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:28:04,907 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:28:09,172 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:28:09,908 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:28:14,173 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:28:14,909 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:28:19,174 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:28:19,910 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:28:24,176 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:28:24,911 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:28:29,176 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:28:29,912 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:28:33,907 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:28:34,177 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:28:35,908 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:28:39,178 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:28:40,909 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:28:44,180 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:28:45,910 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:28:49,181 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:28:50,911 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:28:51,230 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:28:51,231 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:28:51,231 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:28:51,232 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:28:51,313 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:28:54,182 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:28:56,233 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:28:57,271 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:28:57,273 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:28:57,273 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:28:57,273 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:28:57,315 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:28:59,183 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:29:00,316 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:29:01,568 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:29:03,908 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:29:04,184 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:29:06,909 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:29:09,185 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:29:11,910 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:29:14,186 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:29:16,911 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:29:19,187 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:29:21,912 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:29:24,188 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:29:26,913 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:29:29,189 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:29:31,914 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:29:33,909 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:29:34,190 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:29:37,911 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:29:39,191 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:29:42,911 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:29:44,192 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:29:47,912 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:29:49,193 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:29:52,914 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:29:54,194 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:29:55,439 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:29:55,440 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:29:55,440 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:29:55,441 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:29:56,338 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:29:58,442 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:29:59,195 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:30:01,425 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:30:01,426 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:30:01,426 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:30:01,427 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:30:02,340 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:30:03,897 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:30:03,910 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:30:04,196 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:30:04,341 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:30:08,911 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:30:09,196 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:30:13,912 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:30:14,197 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:30:18,913 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:30:19,198 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:30:23,914 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:30:24,199 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:30:28,915 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:30:29,200 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:30:33,911 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:30:34,201 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:30:34,912 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:30:39,202 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:30:39,913 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:30:44,203 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:30:44,914 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:30:49,204 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:30:49,915 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:30:54,206 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:30:54,916 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:30:59,207 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:30:59,918 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:31:00,918 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:31:00,919 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:31:00,919 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:31:00,920 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:31:01,364 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:31:03,912 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:31:04,208 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:31:05,913 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:31:06,908 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:31:06,909 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:31:06,910 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:31:06,910 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:31:07,367 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:31:08,367 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:31:09,209 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:31:11,123 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:31:14,210 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:31:16,124 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:31:19,211 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:31:21,125 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:31:24,212 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:31:26,126 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:31:29,213 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:31:31,127 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:31:33,913 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:31:34,214 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:31:36,914 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:31:39,214 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:31:41,915 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:31:44,215 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:31:46,916 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:31:49,216 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:31:51,917 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:31:54,217 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:31:56,917 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:31:59,218 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:32:01,918 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:32:03,914 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:32:04,219 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:32:07,914 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:32:07,915 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:32:07,916 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:32:07,916 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:32:07,916 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:32:08,389 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:32:09,220 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:32:12,917 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:32:13,908 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:32:13,909 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:32:13,910 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:32:13,910 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:32:14,221 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:32:14,391 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:32:16,392 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:32:18,112 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:32:19,222 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:32:23,113 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:32:24,223 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:32:28,114 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:32:29,224 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:32:33,115 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:32:33,915 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:32:34,225 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:32:38,917 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:32:39,226 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:32:43,918 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:32:44,227 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:32:48,919 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:32:49,229 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:32:53,920 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:32:54,229 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:32:58,921 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:32:59,231 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:33:03,916 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:33:04,232 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:33:04,918 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:33:09,233 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:33:09,919 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:33:12,173 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:33:12,174 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:33:12,174 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:33:12,174 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:33:12,413 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:33:14,234 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:33:15,175 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:33:18,238 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:33:18,240 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:33:18,240 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:33:18,241 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:33:18,415 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:33:19,235 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:33:20,416 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:33:20,601 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:33:24,236 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:33:25,602 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:33:29,237 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:33:30,603 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:33:33,917 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:33:34,238 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:33:35,919 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:33:39,239 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:33:40,920 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:33:44,240 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:33:45,921 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:33:49,241 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:33:50,922 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:33:54,242 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:33:55,923 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:33:59,243 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:34:00,924 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:34:03,918 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:34:04,244 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:34:06,920 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:34:09,245 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:34:11,921 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:34:14,246 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:34:16,922 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:34:19,247 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:34:20,211 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:34:20,213 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:34:20,213 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:34:20,213 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:34:20,438 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:34:22,214 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:34:24,248 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:34:26,175 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:34:26,176 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:34:26,176 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:34:26,177 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:34:26,440 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:34:27,374 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:34:28,441 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:34:29,249 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:34:32,375 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:34:33,919 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:34:34,250 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:34:37,921 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:34:39,251 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:34:42,922 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:34:44,252 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:34:47,923 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:34:49,253 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:34:52,924 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:34:54,254 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:34:57,925 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:34:59,255 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:35:02,926 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:35:03,920 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:35:04,256 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:35:08,922 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:35:09,257 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:35:13,923 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:35:14,258 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:35:18,924 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:35:19,259 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:35:23,925 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:35:24,260 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:35:27,614 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:35:27,616 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:35:27,616 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:35:27,616 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:35:28,468 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:35:29,261 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:35:29,617 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:35:33,621 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:35:33,622 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:35:33,622 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:35:33,622 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:35:33,921 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:35:34,262 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:35:34,471 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:35:34,922 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:35:36,472 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:35:39,263 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:35:39,923 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:35:44,264 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:35:44,924 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:35:49,266 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:35:49,925 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:35:54,267 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:35:54,927 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:35:59,268 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:35:59,928 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:36:03,923 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:36:04,269 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:36:05,924 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:36:09,270 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:36:10,925 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:36:14,271 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:36:15,926 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:36:19,272 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:36:20,927 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:36:24,273 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:36:25,928 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:36:29,274 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:36:30,929 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:36:33,466 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:36:33,467 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:36:33,467 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:36:33,468 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:36:33,493 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:36:33,923 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:36:34,276 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:36:36,924 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:36:39,277 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:36:39,520 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:36:39,522 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:36:39,522 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:36:39,523 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:36:40,496 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:36:42,497 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:36:42,744 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:36:44,278 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:36:47,745 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:36:49,279 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:36:52,746 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:36:54,280 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:36:57,747 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:36:59,281 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:37:02,747 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:37:03,924 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:37:04,282 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:37:07,926 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:37:09,283 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:37:12,927 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:37:14,284 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:37:17,928 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:37:19,285 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:37:22,929 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:37:24,286 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:37:27,930 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:37:29,287 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:37:32,931 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:37:33,926 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:37:34,288 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:37:38,927 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:37:39,289 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:37:40,557 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:37:40,558 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:37:40,558 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:37:40,559 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:37:41,519 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:37:44,290 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:37:44,560 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:37:46,558 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:37:46,560 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:37:46,560 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:37:46,560 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:37:47,521 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:37:48,522 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:37:49,291 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:37:49,761 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:37:54,292 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:37:54,762 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:37:59,294 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:37:59,763 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:38:03,927 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:38:04,295 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:38:04,928 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:38:09,296 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:38:09,929 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:38:14,297 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:38:14,930 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:38:19,298 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:38:19,931 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:38:24,299 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:38:24,932 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:38:29,300 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:38:29,933 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:38:33,928 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:38:34,301 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:38:35,929 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:38:39,303 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:38:40,930 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:38:42,388 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:38:42,389 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:38:42,389 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:38:42,390 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:38:42,543 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:38:44,304 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:38:46,391 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:38:48,326 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:38:48,327 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:38:48,328 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:38:48,328 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:38:48,545 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:38:49,305 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:38:50,546 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:38:51,691 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:38:54,306 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:38:56,693 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:38:59,307 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:39:01,694 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:39:03,929 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:39:04,308 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:39:06,931 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:39:09,309 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:39:11,932 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:39:14,310 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:39:16,933 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:39:19,311 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:39:21,933 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:39:24,312 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:39:26,935 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:39:29,313 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:39:31,936 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:39:33,930 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:39:34,314 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:39:37,932 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:39:39,315 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:39:42,933 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:39:44,316 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:39:47,934 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:39:49,317 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:39:51,364 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:39:51,366 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:39:51,366 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:39:51,366 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:39:51,569 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:39:53,367 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:39:54,318 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:39:57,375 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:39:57,376 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:39:57,376 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:39:57,377 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:39:57,572 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:39:58,572 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:39:58,576 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:39:59,319 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:40:03,577 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:40:03,931 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:40:04,320 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:40:08,933 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:40:09,321 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:40:13,934 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:40:14,322 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:40:18,935 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:40:19,323 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:40:23,936 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:40:24,324 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:40:28,937 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:40:29,325 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:40:33,932 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:40:34,326 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:40:34,933 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:40:39,327 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:40:39,934 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:40:44,328 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:40:44,935 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:40:49,329 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:40:49,936 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:40:54,330 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:40:54,938 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:40:58,439 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:40:58,440 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:40:58,441 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:40:58,441 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:40:58,597 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:40:59,331 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:41:00,442 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:41:03,933 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:41:04,332 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:41:04,425 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:41:04,426 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:41:04,426 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:41:04,427 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:41:04,600 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:41:05,639 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:41:06,601 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:41:09,333 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:41:10,640 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:41:14,334 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:41:15,641 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:41:19,336 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:41:20,641 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:41:24,337 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:41:25,642 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:41:29,338 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:41:30,643 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:41:33,934 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:41:34,339 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:41:35,936 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:41:39,340 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:41:40,937 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:41:44,341 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:41:45,938 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:41:49,342 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:41:50,939 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:41:54,343 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:41:55,940 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:41:59,344 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:42:00,941 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:42:03,935 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:42:04,345 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:42:05,952 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:42:05,954 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:42:05,954 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:42:05,955 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:42:05,955 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:42:06,629 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:42:09,347 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:42:10,957 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:42:12,011 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:42:12,012 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:42:12,013 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:42:12,013 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:42:12,631 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:42:14,348 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:42:14,632 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:42:16,224 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:42:19,349 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:42:21,224 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:42:24,350 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:42:26,226 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:42:29,351 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:42:31,227 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:42:33,936 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:42:34,352 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:42:36,937 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:42:39,353 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:42:41,938 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:42:44,354 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:42:46,939 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:42:49,355 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:42:51,940 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:42:54,356 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:42:56,942 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:42:59,357 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:43:01,944 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:43:03,937 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:43:04,358 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:43:07,939 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:43:09,359 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:43:12,940 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:43:14,360 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:43:14,452 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:43:14,453 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:43:14,453 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:43:14,454 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:43:14,655 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:43:18,455 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:43:19,361 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:43:20,438 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:43:20,440 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:43:20,440 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:43:20,440 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:43:20,657 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:43:22,658 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:43:23,654 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:43:24,362 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:43:28,655 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:43:29,363 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:43:33,655 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:43:33,938 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:43:34,364 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:43:38,940 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:43:39,364 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:43:43,941 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:43:44,365 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:43:48,942 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:43:49,366 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:43:53,943 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:43:54,368 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:43:58,944 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:43:59,369 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:44:03,939 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:44:04,370 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:44:04,940 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:44:09,371 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:44:09,942 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:44:14,372 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:44:14,942 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:44:19,373 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:44:19,943 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:44:22,205 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:44:22,206 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:44:22,206 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:44:22,207 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:44:22,681 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:44:24,374 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:44:25,207 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:44:28,194 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:44:28,195 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:44:28,195 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:44:28,196 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:44:28,683 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:44:29,380 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:44:30,423 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:44:30,684 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:44:33,940 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:44:34,381 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:44:35,941 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:44:39,382 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:44:40,942 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:44:44,383 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:44:45,943 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:44:49,384 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:44:50,944 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:44:54,385 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:44:55,945 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:44:59,386 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:45:00,946 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:45:03,942 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:45:04,387 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:45:06,943 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:45:09,387 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:45:11,943 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:45:14,388 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:45:16,944 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:45:19,389 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:45:21,945 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:45:24,390 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:45:26,946 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:45:27,494 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:45:27,495 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:45:27,496 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:45:27,496 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:45:27,708 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:45:29,391 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:45:32,497 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:45:33,532 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:45:33,534 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:45:33,534 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:45:33,535 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:45:33,711 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:45:33,943 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:45:34,392 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:45:34,712 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:45:37,944 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:45:39,393 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:45:42,945 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:45:44,394 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:45:47,947 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:45:49,395 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:45:52,948 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:45:54,396 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:45:57,949 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:45:59,397 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:46:02,950 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:46:03,944 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:46:04,398 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:46:08,945 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:46:09,399 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:46:13,946 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:46:14,401 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:46:18,947 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:46:19,401 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:46:23,948 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:46:24,402 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:46:28,949 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:46:29,404 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:46:33,528 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:46:33,529 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:46:33,530 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:46:33,530 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:46:33,736 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:46:33,945 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:46:34,405 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:46:34,945 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:46:39,405 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:46:39,519 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:46:39,520 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:46:39,520 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:46:39,521 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:46:39,739 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:46:40,736 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:46:40,740 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:46:44,406 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:46:45,737 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:46:49,407 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:46:50,738 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:46:54,408 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:46:55,739 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:46:59,409 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:47:00,740 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:47:03,945 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:47:04,410 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:47:05,946 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:47:09,411 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:47:10,947 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:47:14,413 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:47:15,948 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:47:19,414 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:47:20,949 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:47:24,415 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:47:25,950 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:47:29,416 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:47:30,951 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:47:33,946 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:47:34,417 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:47:36,948 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:47:39,418 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:47:40,859 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:47:40,861 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:47:40,861 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:47:40,861 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:47:41,764 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:47:42,862 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:47:44,420 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:47:46,816 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:47:46,817 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:47:46,818 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:47:46,818 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:47:47,766 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:47:48,028 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:47:48,767 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:47:49,421 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:47:53,029 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:47:54,422 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:47:58,030 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:47:59,422 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:48:03,031 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:48:03,947 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:48:04,423 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:48:08,948 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:48:09,424 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:48:13,949 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:48:14,425 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:48:18,951 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:48:19,427 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:48:23,951 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:48:24,428 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:48:28,952 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:48:29,429 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:48:33,948 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:48:34,430 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:48:34,949 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:48:39,431 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:48:39,950 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:48:44,432 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:48:44,951 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:48:46,557 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:48:46,559 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:48:46,559 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:48:46,559 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:48:46,790 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:48:49,433 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:48:50,560 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:48:52,526 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:48:52,528 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:48:52,528 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:48:52,528 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:48:52,792 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:48:54,434 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:48:54,793 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:48:55,737 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:48:59,435 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:49:00,738 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:49:03,949 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:49:04,436 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:49:05,950 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:49:09,437 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:49:10,951 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:49:14,437 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:49:15,952 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:49:19,438 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:49:20,953 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:49:24,439 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:49:25,954 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:49:29,440 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:49:30,955 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:49:33,949 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:49:34,441 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:49:36,951 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:49:39,442 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:49:41,952 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:49:44,443 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:49:46,952 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:49:49,444 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:49:51,953 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:49:54,444 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:49:54,597 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:49:54,599 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:49:54,599 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:49:54,600 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:49:54,819 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:49:57,600 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:49:59,445 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:50:00,579 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:50:00,581 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:50:00,581 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:50:00,582 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:50:00,822 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:50:02,857 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:50:03,823 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:50:03,951 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:50:04,446 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:50:07,952 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:50:09,447 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:50:12,953 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:50:14,448 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:50:17,954 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:50:19,449 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:50:22,955 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:50:24,450 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:50:27,956 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:50:29,451 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:50:32,957 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:50:33,952 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:50:34,452 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:50:38,953 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:50:39,453 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:50:43,954 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:50:44,454 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:50:48,955 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:50:49,455 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:50:53,955 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:50:54,456 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:50:58,956 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:50:59,456 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:50:59,749 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:50:59,751 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:50:59,751 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:50:59,751 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:50:59,847 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:51:03,953 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:51:04,457 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:51:04,954 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:51:05,748 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:51:05,749 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:51:05,750 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:51:05,750 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:51:05,849 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:51:07,850 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:51:09,458 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:51:09,980 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:51:14,459 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:51:14,981 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:51:19,460 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:51:19,982 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:51:24,461 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:51:24,983 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:51:29,462 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:51:29,984 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:51:33,953 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:51:34,463 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:51:35,954 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:51:39,464 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:51:40,956 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:51:44,465 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:51:45,956 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:51:49,466 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:51:50,957 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:51:54,466 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:51:55,958 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:51:59,467 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:52:00,959 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:52:03,955 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:52:04,469 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:52:05,680 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:52:05,686 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:52:05,686 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:52:05,686 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:52:05,874 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:52:06,687 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:52:09,469 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:52:11,674 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:52:11,675 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:52:11,676 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:52:11,676 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:52:11,877 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:52:11,888 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:52:13,879 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:52:14,470 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:52:16,889 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:52:19,471 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:52:21,890 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:52:24,472 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:52:26,891 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:52:29,473 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:52:31,891 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:52:33,955 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:52:34,474 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:52:36,956 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:52:39,475 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:52:41,957 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:52:44,476 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:52:46,958 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:52:49,476 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:52:51,959 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:52:54,477 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:52:56,960 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:52:59,478 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:53:01,961 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:53:03,956 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:53:04,479 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:53:07,957 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:53:09,480 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:53:11,429 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:53:11,430 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:53:11,431 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:53:11,431 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:53:11,900 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:53:13,432 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:53:14,480 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:53:17,451 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:53:17,452 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:53:17,452 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:53:17,453 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:53:17,902 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:53:18,648 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:53:19,481 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:53:19,903 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:53:23,649 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:53:24,482 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:53:28,650 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:53:29,483 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:53:33,651 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:53:33,957 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:53:34,484 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:53:38,958 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:53:39,485 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:53:43,959 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:53:44,486 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:53:48,960 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:53:49,486 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:53:53,961 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:53:54,491 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:53:58,962 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:53:59,492 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:54:03,958 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:54:04,493 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:54:04,958 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:54:09,494 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:54:09,959 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:54:14,495 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:54:14,960 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:54:16,551 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:54:16,552 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:54:16,552 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:54:16,553 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:54:16,923 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:54:19,496 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:54:20,554 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:54:22,481 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:54:22,482 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:54:22,482 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:54:22,482 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:54:22,925 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:54:23,926 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:54:24,502 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:54:25,690 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:54:29,503 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:54:30,691 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:54:33,958 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:54:34,504 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:54:35,959 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:54:39,505 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:54:40,960 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:54:44,506 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:54:45,961 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:54:49,507 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:54:50,962 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:54:54,508 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:54:55,963 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:54:59,509 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:55:00,964 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:55:03,960 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:55:04,510 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:55:06,961 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:55:09,511 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:55:11,962 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:55:14,512 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:55:16,963 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:55:19,513 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:55:21,964 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:55:24,110 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:55:24,111 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:55:24,111 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:55:24,112 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:55:24,514 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:55:24,954 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:55:27,113 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:55:29,515 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:55:30,033 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:55:30,035 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:55:30,035 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:55:30,036 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:55:30,957 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:55:31,957 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:55:32,315 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:55:33,961 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:55:34,516 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:55:37,962 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:55:39,517 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:55:42,963 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:55:44,518 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:55:47,964 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:55:49,519 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:55:52,965 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:55:54,520 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:55:57,966 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:55:59,521 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:56:02,966 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:56:03,962 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:56:04,522 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:56:08,963 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:56:09,523 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:56:13,964 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:56:14,524 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:56:18,965 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:56:19,525 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:56:23,966 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:56:24,526 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:56:28,967 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:56:29,527 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:56:32,033 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:56:32,034 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:56:32,035 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:56:32,035 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:56:32,981 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:56:33,963 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:56:34,528 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:56:34,964 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:56:38,045 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:56:38,046 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:56:38,046 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:56:38,047 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:56:38,984 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:56:39,529 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:56:39,984 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:56:40,261 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:56:44,530 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:56:45,262 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:56:49,531 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:56:50,263 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:56:54,532 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:56:55,264 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:56:59,533 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:57:00,265 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:57:03,963 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:57:04,534 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:57:05,965 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:57:09,534 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:57:10,966 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:57:14,536 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:57:15,967 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:57:19,537 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:57:20,968 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:57:24,538 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:57:25,969 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:57:29,539 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:57:30,970 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:57:33,965 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:57:34,540 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:57:36,318 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:57:36,319 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:57:36,320 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:57:36,320 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:57:36,321 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:57:37,007 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:57:39,541 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:57:41,323 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:57:42,273 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:57:42,274 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:57:42,274 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:57:42,275 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:57:43,010 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:57:44,010 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:57:44,541 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:57:46,471 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:57:49,542 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:57:51,472 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:57:54,543 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:57:56,473 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:57:59,544 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:58:01,474 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:58:03,966 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:58:04,545 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:58:06,967 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:58:09,546 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:58:11,968 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:58:14,547 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:58:16,969 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:58:19,548 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:58:21,970 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:58:24,549 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:58:26,971 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:58:29,550 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:58:31,972 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:58:33,967 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:58:34,551 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:58:37,968 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:58:39,552 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:58:41,933 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:58:41,934 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:58:41,934 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:58:41,935 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:58:42,034 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:58:43,935 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:58:44,553 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:58:47,921 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:58:47,922 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:58:47,922 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:58:47,923 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:58:48,036 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:58:49,126 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:58:49,554 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:58:50,037 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:58:54,127 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:58:54,555 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:58:59,128 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:58:59,556 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:59:03,968 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:59:04,558 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:59:04,969 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:59:09,559 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:59:09,970 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:59:14,560 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:59:14,971 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:59:19,561 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:59:19,972 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:59:24,562 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:59:24,973 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:59:29,563 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:59:29,974 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:59:33,969 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 14:59:34,564 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:59:35,970 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:59:39,565 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:59:40,971 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:59:44,566 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:59:45,972 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:59:46,072 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:59:46,073 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:59:46,073 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:59:46,074 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:59:47,061 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:59:49,567 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:59:51,075 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:59:52,055 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 14:59:52,057 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 14:59:52,057 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 14:59:52,058 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 14:59:52,063 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 14:59:54,064 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/output.log +2024-03-14 14:59:54,568 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 14:59:56,315 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 14:59:59,569 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:00:01,316 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:00:03,970 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 15:00:04,570 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:00:06,972 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:00:09,571 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:00:11,973 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:00:14,572 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:00:16,974 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:00:19,573 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:00:21,975 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:00:24,574 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:00:26,975 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:00:29,575 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:00:31,976 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:00:33,971 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 15:00:34,576 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:00:37,972 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:00:39,577 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:00:42,973 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:00:44,578 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:00:47,974 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:00:49,579 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:00:52,975 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:00:54,580 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:00:54,894 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 15:00:54,895 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 15:00:54,895 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 15:00:54,895 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 15:00:55,088 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 15:00:58,896 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:00:59,581 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:01:00,918 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 15:01:00,920 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 15:01:00,920 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 15:01:00,920 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 15:01:01,092 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 15:01:01,133 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: partial_history +2024-03-14 15:01:01,135 DEBUG SenderThread:195 [sender.py:send():382] send: metric +2024-03-14 15:01:01,135 DEBUG SenderThread:195 [sender.py:send():382] send: metric +2024-03-14 15:01:01,136 DEBUG SenderThread:195 [sender.py:send():382] send: metric +2024-03-14 15:01:01,136 DEBUG SenderThread:195 [sender.py:send():382] send: metric +2024-03-14 15:01:01,136 DEBUG SenderThread:195 [sender.py:send():382] send: metric +2024-03-14 15:01:01,136 DEBUG SenderThread:195 [sender.py:send():382] send: history +2024-03-14 15:01:01,136 DEBUG SenderThread:195 [sender.py:send_request():409] send_request: summary_record +2024-03-14 15:01:01,137 INFO SenderThread:195 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-14 15:01:01,138 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: pause +2024-03-14 15:01:01,138 INFO HandlerThread:195 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-14 15:01:01,138 INFO HandlerThread:195 [system_monitor.py:finish():203] Stopping system monitor +2024-03-14 15:01:01,138 DEBUG SystemMonitor:195 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-14 15:01:01,138 DEBUG SystemMonitor:195 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-14 15:01:01,139 INFO HandlerThread:195 [interfaces.py:finish():202] Joined cpu monitor +2024-03-14 15:01:01,140 INFO HandlerThread:195 [interfaces.py:finish():202] Joined disk monitor +2024-03-14 15:01:01,151 INFO HandlerThread:195 [interfaces.py:finish():202] Joined gpu monitor +2024-03-14 15:01:01,151 INFO HandlerThread:195 [interfaces.py:finish():202] Joined memory monitor +2024-03-14 15:01:01,151 INFO HandlerThread:195 [interfaces.py:finish():202] Joined network monitor +2024-03-14 15:01:01,151 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 15:01:02,092 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/wandb-summary.json +2024-03-14 15:01:04,152 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:01:04,582 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:01:09,153 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:01:09,583 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:01:14,154 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:01:14,584 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:01:19,155 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:01:19,585 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:01:23,084 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: resume +2024-03-14 15:01:23,084 INFO HandlerThread:195 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-14 15:01:23,084 INFO HandlerThread:195 [system_monitor.py:start():194] Starting system monitor +2024-03-14 15:01:23,084 INFO SystemMonitor:195 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-14 15:01:23,085 INFO SystemMonitor:195 [interfaces.py:start():190] Started cpu monitoring +2024-03-14 15:01:23,086 INFO SystemMonitor:195 [interfaces.py:start():190] Started disk monitoring +2024-03-14 15:01:23,086 INFO SystemMonitor:195 [interfaces.py:start():190] Started gpu monitoring +2024-03-14 15:01:23,087 INFO SystemMonitor:195 [interfaces.py:start():190] Started memory monitoring +2024-03-14 15:01:23,089 INFO SystemMonitor:195 [interfaces.py:start():190] Started network monitoring +2024-03-14 15:01:23,112 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: pause +2024-03-14 15:01:23,112 INFO HandlerThread:195 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-14 15:01:23,112 INFO HandlerThread:195 [system_monitor.py:finish():203] Stopping system monitor +2024-03-14 15:01:23,112 INFO HandlerThread:195 [interfaces.py:finish():202] Joined cpu monitor +2024-03-14 15:01:23,112 DEBUG SystemMonitor:195 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-14 15:01:23,113 INFO HandlerThread:195 [interfaces.py:finish():202] Joined disk monitor +2024-03-14 15:01:23,113 DEBUG SystemMonitor:195 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-14 15:01:23,113 DEBUG SystemMonitor:195 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-14 15:01:23,129 INFO HandlerThread:195 [interfaces.py:finish():202] Joined gpu monitor +2024-03-14 15:01:23,129 INFO HandlerThread:195 [interfaces.py:finish():202] Joined memory monitor +2024-03-14 15:01:23,130 INFO HandlerThread:195 [interfaces.py:finish():202] Joined network monitor +2024-03-14 15:01:23,130 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 15:01:24,587 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:01:25,136 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:01:26,101 INFO Thread-12 :195 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240314_140503-88c6vfns/files/config.yaml +2024-03-14 15:01:29,588 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:01:30,285 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:01:34,589 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:01:35,286 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:01:39,590 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:01:40,287 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:01:44,591 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:01:45,287 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:01:49,592 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:01:50,288 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:01:54,593 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:01:55,289 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:01:59,594 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:02:00,290 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:02:04,595 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:02:05,291 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report +2024-03-14 15:02:07,706 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: resume +2024-03-14 15:02:07,706 INFO HandlerThread:195 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-14 15:02:07,706 INFO HandlerThread:195 [system_monitor.py:start():194] Starting system monitor +2024-03-14 15:02:07,706 INFO SystemMonitor:195 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-14 15:02:07,707 INFO SystemMonitor:195 [interfaces.py:start():190] Started cpu monitoring +2024-03-14 15:02:07,708 INFO SystemMonitor:195 [interfaces.py:start():190] Started disk monitoring +2024-03-14 15:02:07,708 INFO SystemMonitor:195 [interfaces.py:start():190] Started gpu monitoring +2024-03-14 15:02:07,710 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: pause +2024-03-14 15:02:07,710 INFO HandlerThread:195 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-14 15:02:07,710 INFO HandlerThread:195 [system_monitor.py:finish():203] Stopping system monitor +2024-03-14 15:02:07,710 INFO HandlerThread:195 [interfaces.py:finish():202] Joined cpu monitor +2024-03-14 15:02:07,711 INFO SystemMonitor:195 [interfaces.py:start():190] Started memory monitoring +2024-03-14 15:02:07,711 DEBUG SystemMonitor:195 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-14 15:02:07,711 DEBUG SystemMonitor:195 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-14 15:02:07,711 DEBUG SystemMonitor:195 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-14 15:02:07,713 INFO HandlerThread:195 [interfaces.py:finish():202] Joined disk monitor +2024-03-14 15:02:07,733 INFO HandlerThread:195 [interfaces.py:finish():202] Joined gpu monitor +2024-03-14 15:02:07,733 INFO HandlerThread:195 [interfaces.py:finish():202] Joined memory monitor +2024-03-14 15:02:07,734 DEBUG SenderThread:195 [sender.py:send():382] send: stats +2024-03-14 15:02:09,596 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: keepalive +2024-03-14 15:02:10,342 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: resume +2024-03-14 15:02:10,342 INFO HandlerThread:195 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-14 15:02:10,342 INFO HandlerThread:195 [system_monitor.py:start():194] Starting system monitor +2024-03-14 15:02:10,342 INFO SystemMonitor:195 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-14 15:02:10,343 INFO SystemMonitor:195 [interfaces.py:start():190] Started cpu monitoring +2024-03-14 15:02:10,343 INFO SystemMonitor:195 [interfaces.py:start():190] Started disk monitoring +2024-03-14 15:02:10,344 INFO SystemMonitor:195 [interfaces.py:start():190] Started gpu monitoring +2024-03-14 15:02:10,346 INFO SystemMonitor:195 [interfaces.py:start():190] Started memory monitoring +2024-03-14 15:02:10,347 INFO SystemMonitor:195 [interfaces.py:start():190] Started network monitoring +2024-03-14 15:02:10,734 DEBUG HandlerThread:195 [handler.py:handle_request():146] handle_request: status_report