diff --git "a/wandb/run-20240411_040722-rimyh9a1/logs/debug-internal.log" "b/wandb/run-20240411_040722-rimyh9a1/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20240411_040722-rimyh9a1/logs/debug-internal.log" @@ -0,0 +1,15349 @@ +2024-04-11 04:07:22,850 INFO StreamThr :1091 [internal.py:wandb_internal():86] W&B internal server running at pid: 1091, started at: 2024-04-11 04:07:22.849871 +2024-04-11 04:07:22,852 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status +2024-04-11 04:07:23,106 INFO WriterThread:1091 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/run-rimyh9a1.wandb +2024-04-11 04:07:23,107 DEBUG SenderThread:1091 [sender.py:send():379] send: header +2024-04-11 04:07:23,109 DEBUG SenderThread:1091 [sender.py:send():379] send: run +2024-04-11 04:07:23,229 INFO SenderThread:1091 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files +2024-04-11 04:07:23,229 INFO SenderThread:1091 [sender.py:_start_run_threads():1124] run started: rimyh9a1 with start time 1712808442.848646 +2024-04-11 04:07:23,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: check_version +2024-04-11 04:07:23,236 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: check_version +2024-04-11 04:07:23,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: run_start +2024-04-11 04:07:23,341 DEBUG HandlerThread:1091 [system_info.py:__init__():26] System info init +2024-04-11 04:07:23,341 DEBUG HandlerThread:1091 [system_info.py:__init__():41] System info init done +2024-04-11 04:07:23,341 INFO HandlerThread:1091 [system_monitor.py:start():194] Starting system monitor +2024-04-11 04:07:23,341 INFO SystemMonitor:1091 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-11 04:07:23,342 INFO HandlerThread:1091 [system_monitor.py:probe():214] Collecting system info +2024-04-11 04:07:23,342 INFO SystemMonitor:1091 [interfaces.py:start():190] Started cpu monitoring +2024-04-11 04:07:23,343 INFO SystemMonitor:1091 [interfaces.py:start():190] Started disk monitoring +2024-04-11 04:07:23,344 INFO SystemMonitor:1091 [interfaces.py:start():190] Started gpu monitoring +2024-04-11 04:07:23,345 INFO SystemMonitor:1091 [interfaces.py:start():190] Started memory monitoring +2024-04-11 04:07:23,346 INFO SystemMonitor:1091 [interfaces.py:start():190] Started network monitoring +2024-04-11 04:07:23,361 DEBUG HandlerThread:1091 [system_info.py:probe():150] Probing system +2024-04-11 04:07:23,364 DEBUG HandlerThread:1091 [gitlib.py:_init_repo():56] git repository is invalid +2024-04-11 04:07:23,364 DEBUG HandlerThread:1091 [system_info.py:probe():198] Probing system done +2024-04-11 04:07:23,364 DEBUG HandlerThread:1091 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-04-11T04:07:23.361534', 'startedAt': '2024-04-11T04:07:22.840819', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': '321ddc84c049', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.19, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.19, 'min': 0.0, 'max': 0.0}, {'current': 2000.19, 'min': 0.0, 'max': 0.0}, {'current': 2000.19, 'min': 0.0, 'max': 0.0}, {'current': 2000.19, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5566.962131500244}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.357559204101562}} +2024-04-11 04:07:23,364 INFO HandlerThread:1091 [system_monitor.py:probe():224] Finished collecting system info +2024-04-11 04:07:23,364 INFO HandlerThread:1091 [system_monitor.py:probe():227] Publishing system info +2024-04-11 04:07:23,364 DEBUG HandlerThread:1091 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-04-11 04:07:24,231 INFO Thread-12 :1091 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/conda-environment.yaml +2024-04-11 04:07:38,380 ERROR HandlerThread:1091 [system_info.py:_save_conda():221] Error saving conda packages: Command '['conda', 'env', 'export']' timed out after 15 seconds +Traceback (most recent call last): + File "/opt/conda/lib/python3.10/site-packages/wandb/sdk/internal/system/system_info.py", line 214, in _save_conda + subprocess.call( + File "/opt/conda/lib/python3.10/subprocess.py", line 347, in call + return p.wait(timeout=timeout) + File "/opt/conda/lib/python3.10/subprocess.py", line 1209, in wait + return self._wait(timeout=timeout) + File "/opt/conda/lib/python3.10/subprocess.py", line 1951, in _wait + raise TimeoutExpired(self.args, timeout) +subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after 15 seconds +2024-04-11 04:07:38,381 DEBUG HandlerThread:1091 [system_info.py:_save_conda():222] Saving conda packages done +2024-04-11 04:07:38,381 INFO HandlerThread:1091 [system_monitor.py:probe():229] Finished publishing system info +2024-04-11 04:07:38,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:07:38,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 04:07:38,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:07:38,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 04:07:38,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:07:38,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 04:07:38,388 DEBUG SenderThread:1091 [sender.py:send():379] send: files +2024-04-11 04:07:38,388 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-metadata.json with policy now +2024-04-11 04:07:38,590 INFO wandb-upload_0:1091 [upload_job.py:push():131] Uploaded file /tmp/tmppwkfxjn1wandb/7mpz5val-wandb-metadata.json +2024-04-11 04:07:39,233 INFO Thread-12 :1091 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-metadata.json +2024-04-11 04:07:39,316 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: python_packages +2024-04-11 04:07:39,317 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: python_packages +2024-04-11 04:07:39,319 DEBUG SenderThread:1091 [sender.py:send():379] send: telemetry +2024-04-11 04:07:39,329 DEBUG SenderThread:1091 [sender.py:send():379] send: config +2024-04-11 04:07:39,330 DEBUG SenderThread:1091 [sender.py:send():379] send: metric +2024-04-11 04:07:39,331 DEBUG SenderThread:1091 [sender.py:send():379] send: telemetry +2024-04-11 04:07:39,332 DEBUG SenderThread:1091 [sender.py:send():379] send: metric +2024-04-11 04:07:39,332 WARNING SenderThread:1091 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) +2024-04-11 04:07:39,332 DEBUG SenderThread:1091 [sender.py:send():379] send: telemetry +2024-04-11 04:07:39,333 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:07:39,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:07:39,335 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:07:40,234 INFO Thread-12 :1091 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:07:40,234 INFO Thread-12 :1091 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/requirements.txt +2024-04-11 04:07:42,234 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:07:43,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:07:44,235 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:07:46,236 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:07:48,901 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:07:53,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:07:54,239 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/config.yaml +2024-04-11 04:07:55,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:07:55,239 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:07:55,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:07:59,317 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:04,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:09,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:09,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:08:09,452 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:08:09,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:08:15,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:20,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:23,346 DEBUG SystemMonitor:1091 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-11 04:08:23,348 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:08:24,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:08:24,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:08:24,495 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:08:26,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:28,050 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:08:28,053 DEBUG SenderThread:1091 [sender.py:send():379] send: metric +2024-04-11 04:08:28,053 DEBUG SenderThread:1091 [sender.py:send():379] send: metric +2024-04-11 04:08:28,053 DEBUG SenderThread:1091 [sender.py:send():379] send: metric +2024-04-11 04:08:28,054 DEBUG SenderThread:1091 [sender.py:send():379] send: metric +2024-04-11 04:08:28,054 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:08:28,054 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:08:28,054 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:08:28,255 INFO Thread-12 :1091 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:08:30,256 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:08:31,348 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:34,258 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:08:36,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:39,455 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:08:39,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:08:39,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:08:42,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:47,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:52,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:53,349 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:08:54,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:08:54,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:08:54,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:08:58,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:08:59,267 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/config.yaml +2024-04-11 04:09:03,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:08,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:09,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:09:09,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:09:09,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:09:12,733 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:09:12,735 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:09:12,735 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:09:12,737 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:09:13,272 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:09:13,774 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:14,273 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:09:18,274 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:09:19,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:23,350 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:09:24,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:09:24,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:09:24,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:09:24,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:30,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:35,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:39,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:09:39,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:09:39,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:09:41,114 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:09:41,116 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:09:41,116 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:09:41,116 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:41,119 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:09:41,284 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:09:44,285 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:09:46,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:46,286 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:09:51,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:09:53,351 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:09:54,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:09:54,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:09:54,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:09:57,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:02,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:07,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:09,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:10:09,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:10:09,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:10:10,746 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:10:10,747 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:10:10,748 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:10:10,750 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:10:11,295 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:10:12,295 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:10:12,787 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:16,297 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:10:17,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:22,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:23,352 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:10:24,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:10:24,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:10:24,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:10:28,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:33,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:38,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:39,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:10:39,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:10:39,473 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:10:40,951 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:10:40,953 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:10:40,953 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:10:40,955 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:10:41,307 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:10:42,308 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:10:43,974 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:46,309 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:10:49,699 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:10:53,353 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:10:54,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:10:54,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:10:54,473 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:10:55,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:00,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:05,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:09,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:11:09,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:11:09,473 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:11:11,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:15,550 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:11:15,551 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:11:15,551 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:11:15,553 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:11:16,321 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:11:16,591 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:18,322 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:11:20,323 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:11:22,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:22,324 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:11:23,354 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:11:24,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:11:24,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:11:24,473 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:11:28,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:33,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:38,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:39,459 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:11:39,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:11:39,473 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:11:40,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:11:44,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:49,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:51,547 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:11:51,548 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:11:51,548 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:11:51,549 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:11:52,336 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:11:53,354 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:11:54,337 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:11:54,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:11:54,474 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:11:54,474 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:11:54,475 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:11:56,338 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:12:00,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:05,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:05,935 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:12:05,936 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:12:05,937 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:12:05,937 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:12:06,342 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:12:08,343 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:12:09,459 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:12:09,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:12:09,474 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:12:10,344 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:12:11,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:16,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:21,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:23,355 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:12:24,459 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:12:24,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:12:24,474 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:12:27,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:31,047 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:12:31,049 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:12:31,049 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:12:31,049 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:12:31,352 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:12:32,352 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:12:33,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:36,354 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:12:38,317 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:39,485 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:12:39,486 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:12:39,489 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:12:44,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:47,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:12:47,842 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:12:47,842 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:12:47,842 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:12:48,359 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:12:49,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:12:50,359 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:12:52,360 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:12:53,356 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:12:54,490 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:12:54,491 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:12:54,491 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:12:55,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:00,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:05,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:07,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:13:07,619 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:13:07,619 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:13:07,619 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:13:08,366 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:13:10,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:13:10,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:13:10,207 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:13:10,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:10,367 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:13:12,368 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:13:15,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:20,825 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:21,255 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:13:21,256 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:13:21,256 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:13:21,258 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:13:21,371 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:13:22,371 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:13:23,358 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:13:25,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:13:25,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:13:25,210 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:13:26,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:26,373 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:13:31,301 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:36,302 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:39,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:13:39,561 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:13:39,561 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:13:39,562 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:13:40,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:13:40,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:13:40,211 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:13:40,378 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:13:42,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:42,379 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:13:44,380 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:13:47,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:52,708 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:13:53,358 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:13:55,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:13:55,211 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:13:55,211 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:13:55,393 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:13:55,394 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:13:55,394 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:13:55,395 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:13:56,392 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:13:58,392 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:13:58,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:00,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:14:03,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:08,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:10,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:14:10,211 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:14:10,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:14:14,286 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:15,975 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:14:15,976 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:14:15,977 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:14:15,977 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:14:16,399 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:14:18,400 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:14:20,122 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:20,401 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:14:23,360 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:14:25,212 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:14:25,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:14:25,213 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:14:25,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:29,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:14:29,755 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:14:29,756 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:14:29,756 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:14:30,405 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:14:30,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:32,406 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:14:34,407 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:14:36,163 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:40,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:14:40,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:14:40,215 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:14:41,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:46,357 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:49,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:14:49,455 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:14:49,456 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:14:49,456 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:14:50,413 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:14:51,488 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:14:52,413 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:14:53,360 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:14:54,414 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:14:55,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:14:55,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:14:55,214 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:14:57,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:02,332 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:05,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:15:05,935 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:15:05,936 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:15:05,936 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:15:06,418 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:15:07,970 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:08,419 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:15:10,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:15:10,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:15:10,215 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:15:10,420 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:15:13,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:18,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:22,439 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:15:22,440 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:15:22,441 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:15:22,441 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:15:23,361 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:15:23,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:23,424 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:15:24,425 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:15:25,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:15:25,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:15:25,214 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:15:26,426 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:15:28,426 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:15:28,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:33,575 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:38,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:40,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:15:40,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:15:40,218 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:15:40,338 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:15:40,339 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:15:40,339 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:15:40,340 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:15:40,431 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:15:42,432 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:15:44,433 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:15:44,478 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:49,479 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:53,362 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:15:55,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:15:55,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:15:55,216 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:15:55,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:15:57,762 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:15:57,763 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:15:57,764 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:15:57,764 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:15:58,438 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:16:00,439 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:16:00,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:02,440 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:16:06,155 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:10,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:16:10,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:16:10,215 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:16:11,320 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:16,321 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:16,566 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:16:16,568 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:16:16,568 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:16:16,568 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:16:17,446 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:16:18,447 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:16:21,962 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:22,448 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:16:23,363 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:16:25,217 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:16:25,217 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:16:25,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:16:27,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:32,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:16:32,171 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:16:32,171 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:16:32,172 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:16:32,452 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:16:33,203 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:34,453 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:16:36,453 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:16:38,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:40,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:16:40,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:16:40,217 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:16:44,302 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:49,303 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:53,364 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:16:54,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:16:54,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:16:54,714 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:16:54,714 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:16:54,715 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:16:55,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:16:55,217 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:16:55,217 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:16:55,460 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:16:56,460 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:16:59,878 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:00,461 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:17:04,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:09,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:10,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:17:10,219 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:17:10,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:17:11,029 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:17:11,031 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:17:11,031 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:17:11,031 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:17:11,466 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:17:12,466 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:17:15,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:16,468 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:17:20,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:23,365 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:17:25,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:17:25,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:17:25,220 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:17:25,299 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:28,162 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:17:28,164 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:17:28,164 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:17:28,164 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:17:28,472 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:17:30,473 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:17:31,203 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:34,474 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:17:37,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:40,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:17:40,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:17:40,220 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:17:42,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:46,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:17:46,044 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:17:46,044 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:17:46,044 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:17:46,479 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:17:48,083 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:48,480 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:17:50,480 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:17:53,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:17:53,367 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:17:55,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:17:55,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:17:55,219 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:17:58,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:03,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:05,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:18:05,870 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:18:05,870 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:18:05,871 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:18:06,486 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:18:08,487 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:18:08,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:10,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:18:10,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:18:10,220 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:18:10,488 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:18:14,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:19,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:23,367 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:18:24,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:24,895 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:18:24,896 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:18:24,896 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:18:24,896 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:18:25,217 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:18:25,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:18:25,218 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:18:25,494 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:18:26,494 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:18:30,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:30,496 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:18:35,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:40,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:40,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:18:40,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:18:40,220 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:18:45,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:46,064 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:18:46,066 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:18:46,066 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:18:46,066 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:18:46,502 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:18:48,503 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:18:50,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:18:50,504 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:18:53,368 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:18:55,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:18:55,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:18:55,220 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:18:56,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:01,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:04,874 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:19:04,875 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:19:04,875 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:19:04,876 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:19:05,510 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:19:06,510 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:19:06,915 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:10,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:19:10,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:19:10,222 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:19:10,512 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:19:12,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:17,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:22,306 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:22,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:19:22,759 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:19:22,759 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:19:22,759 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:19:23,369 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:19:23,516 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:19:24,516 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:19:25,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:19:25,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:19:25,352 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:19:26,517 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:19:27,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:28,518 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:19:32,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:37,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:39,840 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:19:39,841 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:19:39,841 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:19:39,842 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:19:40,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:19:40,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:19:40,350 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:19:40,522 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:19:40,522 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:19:42,963 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:44,523 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:19:47,964 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:52,965 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:19:53,370 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:19:55,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:19:55,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:19:55,353 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:19:58,426 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:00,023 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:20:00,025 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:20:00,025 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:20:00,025 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:20:00,529 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:20:02,530 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:20:04,139 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:04,531 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:20:09,140 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:10,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:20:10,350 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:20:10,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:20:14,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:19,477 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:19,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:20:19,873 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:20:19,874 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:20:19,874 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:20:20,536 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:20:20,537 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:20:23,371 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:20:24,538 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:20:25,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:25,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:20:25,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:20:25,353 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:20:30,421 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:35,422 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:36,285 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:20:36,286 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:20:36,286 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:20:36,287 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:20:36,542 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:20:38,543 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:20:40,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:20:40,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:20:40,401 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:20:40,480 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:40,544 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:20:45,481 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:50,482 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:52,289 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:20:52,290 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:20:52,290 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:20:52,290 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:20:52,548 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:20:53,372 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:20:54,549 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:20:55,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:20:55,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:20:55,369 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:20:55,528 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:20:56,549 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:21:00,529 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:05,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:10,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:21:10,354 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:21:10,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:21:10,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:21:11,000 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:21:11,000 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:21:11,000 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:11,001 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:21:11,555 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:21:12,556 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:21:14,556 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:21:16,143 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:16,557 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:21:21,144 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:23,373 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:21:25,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:21:25,354 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:21:25,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:21:26,429 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:31,430 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:32,405 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:21:32,406 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:21:32,406 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:21:32,406 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:21:32,568 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:21:34,569 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:21:36,521 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:36,570 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:21:40,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:21:40,354 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:21:40,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:21:42,417 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:47,418 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:52,419 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:53,374 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:21:55,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:21:55,354 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:21:55,395 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:21:57,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:21:57,081 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:21:57,081 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:21:57,082 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:21:57,578 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:21:58,120 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:21:58,578 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:22:02,580 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:22:03,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:08,292 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:10,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:22:10,355 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:22:10,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:22:13,389 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:18,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:21,793 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:22:21,794 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:22:21,794 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:22:21,795 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:22:22,587 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:22:22,587 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:22:23,375 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:22:24,376 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:25,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:22:25,354 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:22:25,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:22:26,588 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:22:29,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:34,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:36,441 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:22:36,442 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:22:36,442 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:22:36,443 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:22:36,592 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:22:38,593 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:22:39,470 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:40,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:22:40,355 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:22:40,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:22:40,593 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:22:45,393 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:50,394 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:53,376 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:22:55,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:22:55,088 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:22:55,088 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:22:55,088 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:22:55,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:22:55,355 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:22:55,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:22:55,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:22:55,599 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:22:56,600 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:22:58,601 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:00,601 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:01,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:06,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:10,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:23:10,357 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:23:10,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:23:11,467 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:16,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:18,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:23:18,859 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:23:18,859 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:23:18,860 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:23:19,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:23:20,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:21,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:22,609 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:23,377 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:23:25,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:23:25,355 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:23:25,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:23:27,374 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:32,375 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:35,183 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:23:35,184 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:23:35,185 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:23:35,185 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:23:35,614 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:23:36,615 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:38,177 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:38,616 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:40,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:23:40,355 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:23:40,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:23:40,616 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:43,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:48,436 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:52,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:23:52,722 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:23:52,723 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:23:52,723 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:23:53,378 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:23:53,634 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:23:54,379 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:23:54,635 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:55,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:23:55,359 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:23:55,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:23:56,635 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:23:59,924 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:04,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:09,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:10,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:24:10,361 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:24:10,430 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:24:13,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:24:13,435 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:24:13,435 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:24:13,436 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:24:13,642 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:24:15,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:15,643 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:24:19,644 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:24:20,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:23,380 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:24:25,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:24:25,358 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:24:25,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:24:26,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:31,267 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:24:31,269 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:24:31,269 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:24:31,269 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:24:31,648 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:24:32,308 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:33,649 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:24:35,650 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:24:37,417 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:37,651 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:24:40,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:24:40,358 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:24:40,430 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:24:42,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:47,429 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:48,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:24:48,190 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:24:48,190 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:24:48,190 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:24:48,655 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:24:49,655 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:24:53,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:24:53,380 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:24:53,657 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:24:55,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:24:55,359 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:24:55,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:24:58,443 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:03,444 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:04,097 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:25:04,099 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:25:04,099 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:25:04,099 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:25:04,660 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:25:05,661 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:25:09,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:09,662 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:25:10,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:25:10,359 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:25:10,429 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:25:14,395 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:19,396 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:19,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:25:19,821 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:25:19,821 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:25:19,822 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:25:20,666 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:25:21,667 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:25:23,381 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:25:25,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:25:25,359 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:25:25,401 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:25,430 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:25:25,668 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:25:30,402 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:35,402 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:40,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:25:40,361 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:25:40,414 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:40,429 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:25:40,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:25:40,668 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:25:40,668 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:25:40,669 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:25:40,674 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:25:43,675 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:25:45,676 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:25:45,804 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:50,804 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:53,382 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:25:55,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:25:55,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:25:55,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:25:56,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:25:56,384 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:25:56,385 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:25:56,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:25:56,386 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:25:56,680 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:25:57,680 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:26:01,516 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:01,682 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:26:06,517 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:10,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:26:10,361 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:26:10,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:26:11,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:26:11,758 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:26:11,758 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:26:11,758 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:11,759 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:26:12,686 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:26:13,686 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:26:16,885 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:17,688 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:26:21,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:23,383 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:26:25,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:26:25,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:26:25,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:26:26,636 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:26:26,637 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:26:26,638 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:26:26,638 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:26:26,691 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:26:27,677 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:29,692 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:26:31,693 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:26:32,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:37,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:40,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:26:40,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:26:40,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:26:43,396 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:48,397 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:49,130 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:26:49,132 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:26:49,132 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:26:49,132 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:26:49,699 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:26:51,700 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:26:53,384 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:26:53,701 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:26:54,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:26:55,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:26:55,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:26:55,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:26:59,444 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:04,445 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:05,053 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:27:05,054 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:27:05,055 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:27:05,055 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:27:05,705 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:27:07,706 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:27:09,707 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:27:10,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:10,361 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:27:10,361 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:27:10,436 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:27:15,445 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:20,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:20,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:27:20,615 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:27:20,616 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:27:20,616 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:27:20,711 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:27:23,386 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:27:23,712 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:27:25,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:27:25,363 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:27:25,436 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:27:25,713 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:27:26,415 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:31,415 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:36,416 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:39,248 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:27:39,249 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:27:39,249 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:27:39,250 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:27:39,718 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:27:40,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:27:40,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:27:40,949 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:27:41,718 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:27:42,040 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:43,721 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:27:47,333 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:52,333 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:54,276 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:27:55,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:27:55,948 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:27:55,951 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:27:58,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:27:59,292 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:27:59,294 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:27:59,294 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:27:59,294 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:27:59,725 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:28:01,726 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:28:03,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:03,727 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:28:08,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:11,125 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:28:11,126 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:28:11,166 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:28:14,222 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:16,787 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:28:16,788 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:28:16,788 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:28:16,789 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:28:17,732 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:28:19,733 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:28:19,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:21,734 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:28:23,388 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:28:25,389 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:26,077 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:28:26,078 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:28:26,082 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:28:31,175 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:33,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:28:33,817 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:28:33,818 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:28:33,819 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:28:34,739 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:28:35,739 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:28:36,850 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:39,741 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:28:41,078 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:28:41,079 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:28:41,083 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:28:42,140 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:47,141 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:52,142 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:53,388 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:28:54,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:28:54,538 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:28:54,539 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:28:54,539 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:28:54,746 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:28:55,747 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:28:56,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:28:56,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:28:56,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:28:57,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:28:59,749 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:29:02,733 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:07,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:11,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:29:11,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:29:11,278 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:29:13,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:13,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:29:13,455 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:29:13,455 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:29:13,457 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:29:13,755 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:29:15,756 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:29:17,756 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:29:18,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:23,390 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:29:24,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:26,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:29:26,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:29:26,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:29:29,398 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:31,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:29:31,362 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:29:31,362 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:29:31,362 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:29:31,762 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:29:33,762 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:29:34,403 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:35,763 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:29:37,764 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:29:39,809 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:41,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:29:41,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:29:41,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:29:45,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:50,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:53,391 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:29:55,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:29:56,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:29:56,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:29:56,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:29:56,403 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:29:56,404 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:29:56,405 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:29:56,405 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:29:56,773 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:29:57,773 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:30:00,516 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:01,775 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:30:05,517 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:10,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:11,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:30:11,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:30:11,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:30:16,376 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:18,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:30:18,359 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:30:18,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:30:18,360 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:30:18,781 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:30:19,781 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:30:21,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:23,392 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:30:23,783 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:30:26,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:30:26,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:30:26,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:30:27,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:32,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:37,370 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:37,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:30:37,742 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:30:37,742 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:30:37,742 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:30:37,788 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:30:39,789 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:30:41,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:30:41,280 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:30:41,280 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:30:41,790 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:30:42,396 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:47,397 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:52,398 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:53,326 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:30:53,327 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:30:53,327 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:30:53,328 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:30:53,393 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:30:53,794 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:30:55,795 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:30:56,299 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:30:56,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:30:56,300 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:30:57,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:30:57,796 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:31:02,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:07,535 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:09,123 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:31:09,124 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:31:09,124 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:31:09,125 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:31:09,801 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:31:11,747 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:31:11,748 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:31:11,748 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:31:11,802 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:31:13,286 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:13,802 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:31:18,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:23,288 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:23,394 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:31:26,746 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:31:26,747 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:31:26,747 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:31:28,851 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:30,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:31:30,670 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:31:30,670 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:31:30,671 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:31:30,809 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:31:31,809 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:31:34,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:35,811 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:31:39,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:41,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:31:41,746 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:31:41,749 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:31:45,789 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:48,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:31:48,681 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:31:48,682 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:31:48,682 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:31:48,816 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:31:49,816 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:31:51,714 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:31:53,395 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:31:53,818 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:31:56,748 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:31:56,748 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:31:56,752 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:31:56,850 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:01,851 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:03,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:32:03,003 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:32:03,003 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:32:03,004 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:32:03,822 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:32:05,824 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:32:07,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:07,824 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:32:11,749 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:32:11,749 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:32:11,750 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:32:12,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:17,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:20,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:32:20,690 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:32:20,690 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:32:20,691 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:32:20,829 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:32:21,829 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:32:23,396 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:32:23,397 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:25,831 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:32:26,746 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:32:26,747 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:32:26,747 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:32:28,834 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:33,834 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:38,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:41,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:32:41,784 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:32:41,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:32:42,483 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:32:42,484 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:32:42,485 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:32:42,485 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:32:42,838 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:32:43,838 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:32:44,524 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:47,840 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:32:49,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:53,402 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:32:55,403 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:32:56,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:32:56,783 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:32:56,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:33:00,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:01,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:33:01,081 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:33:01,082 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:33:01,082 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:33:01,845 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:33:03,846 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:33:05,847 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:33:06,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:11,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:11,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:33:11,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:33:11,783 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:33:16,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:19,150 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:33:19,151 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:33:19,152 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:33:19,152 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:33:19,852 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:33:21,853 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:33:22,191 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:23,402 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:33:23,854 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:33:26,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:33:26,782 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:33:26,785 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:33:27,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:32,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:36,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:33:36,029 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:33:36,029 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:33:36,031 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:33:36,860 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:33:37,860 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:33:38,068 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:39,861 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:33:41,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:33:41,783 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:33:41,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:33:43,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:48,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:53,402 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:33:54,403 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:33:56,465 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:33:56,467 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:33:56,467 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:33:56,467 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:33:56,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:33:56,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:33:56,783 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:33:56,867 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:33:57,868 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:33:59,432 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:01,869 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:34:04,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:09,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:11,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:34:11,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:34:11,784 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:34:14,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:16,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:34:16,244 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:34:16,244 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:34:16,246 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:34:16,874 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:34:17,875 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:34:20,346 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:21,876 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:34:23,403 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:34:25,404 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:26,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:34:26,784 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:34:26,787 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:34:30,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:32,977 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:34:32,979 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:34:32,979 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:34:32,979 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:34:33,881 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:34:33,881 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:34:35,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:37,883 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:34:41,099 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:41,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:34:41,783 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:34:41,785 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:34:46,901 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:51,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:53,405 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:34:55,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:34:55,216 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:34:55,216 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:34:55,217 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:34:55,889 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:34:55,890 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:34:56,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:34:56,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:34:56,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:34:56,938 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:34:59,891 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:35:02,459 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:07,460 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:11,935 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:35:11,936 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:35:11,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:35:13,023 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:13,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:35:13,231 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:35:13,232 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:35:13,232 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:35:13,896 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:35:13,897 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:35:17,898 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:35:18,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:23,408 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:35:24,409 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:26,961 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:35:26,961 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:35:26,962 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:35:29,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:34,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:37,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:35:37,361 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:35:37,362 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:35:37,362 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:35:37,905 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:35:39,906 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:35:40,370 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:41,906 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:35:41,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:35:41,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:35:41,941 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:35:46,076 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:51,077 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:53,409 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:35:56,410 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:35:56,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:35:56,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:35:56,942 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:35:58,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:35:58,531 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:35:58,531 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:35:58,532 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:35:58,913 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:35:59,914 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:36:01,563 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:03,916 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:36:06,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:11,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:11,968 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:36:11,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:36:11,969 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:36:15,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:36:15,207 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:36:15,207 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:36:15,208 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:36:15,921 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:36:15,921 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:36:17,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:19,922 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:36:22,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:23,410 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:36:26,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:36:26,946 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:36:26,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:36:28,034 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:33,035 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:35,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:36:35,940 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:36:35,940 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:36:35,941 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:36:36,929 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:36:37,929 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:36:38,933 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:39,930 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:36:41,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:36:41,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:36:41,946 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:36:44,047 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:49,048 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:53,411 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:36:54,411 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:54,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:36:54,674 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:36:54,674 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:36:54,675 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:36:54,935 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:36:55,936 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:36:57,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:36:57,274 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:36:57,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:36:59,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:36:59,940 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:37:04,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:09,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:12,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:37:12,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:37:12,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:37:15,317 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:17,850 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:37:17,851 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:37:17,851 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:37:17,852 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:37:17,946 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:37:19,947 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:37:20,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:21,948 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:37:23,411 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:37:26,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:27,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:37:27,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:37:27,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:37:32,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:37:32,218 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:37:32,218 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:37:32,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:32,219 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:37:32,953 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:37:33,953 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:37:35,954 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:37:37,341 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:42,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:37:42,274 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:37:42,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:37:42,399 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:47,400 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:49,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:37:49,329 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:37:49,329 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:37:49,330 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:37:49,959 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:37:49,960 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:37:52,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:37:53,417 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:37:53,961 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:37:57,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:37:57,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:37:57,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:37:58,397 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:03,397 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:08,398 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:08,409 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:38:08,410 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:38:08,410 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:38:08,410 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:38:08,967 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:38:09,967 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:38:11,968 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:38:12,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:38:12,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:38:12,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:38:13,969 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:38:14,397 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:19,398 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:22,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:38:22,545 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:38:22,545 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:38:22,546 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:38:22,972 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:38:23,413 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:38:23,973 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:38:24,414 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:25,974 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:38:27,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:38:27,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:38:27,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:38:27,974 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:38:30,410 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:35,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:38,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:38:38,046 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:38:38,046 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:38:38,047 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:38:38,979 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:38:39,979 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:38:41,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:41,980 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:38:42,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:38:42,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:38:42,280 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:38:46,411 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:51,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:53,414 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:38:56,415 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:38:57,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:38:57,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:38:57,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:38:59,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:38:59,577 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:38:59,578 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:38:59,580 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:38:59,987 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:38:59,987 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:39:01,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:03,988 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:39:06,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:11,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:12,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:39:12,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:39:12,278 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:39:14,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:39:14,583 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:39:14,583 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:39:14,583 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:39:14,992 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:39:15,993 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:39:17,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:17,994 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:39:19,995 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:39:22,732 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:23,415 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:39:27,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:39:27,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:39:27,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:39:28,346 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:33,347 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:37,111 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:39:37,112 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:39:37,112 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:39:37,115 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:39:38,001 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:39:38,001 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:39:39,152 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:42,002 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:39:42,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:39:42,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:39:42,281 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:39:44,380 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:49,381 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:53,416 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:39:53,763 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:39:53,764 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:39:53,764 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:39:53,765 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:39:54,007 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:39:54,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:39:56,008 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:39:57,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:39:57,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:39:57,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:39:58,009 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:40:00,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:05,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:10,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:11,776 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:40:11,777 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:40:11,777 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:40:11,779 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:40:12,015 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:40:12,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:40:12,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:40:12,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:40:14,016 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:40:15,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:16,016 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:40:20,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:23,417 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:40:26,418 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:27,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:40:27,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:40:27,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:40:31,427 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:33,479 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:40:33,480 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:40:33,480 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:40:33,481 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:40:34,023 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:40:34,023 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:40:36,520 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:38,025 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:40:41,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:42,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:40:42,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:40:42,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:40:47,314 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:50,442 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:40:50,443 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:40:50,443 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:40:50,445 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:40:51,030 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:40:52,030 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:40:52,483 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:40:53,418 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:40:54,031 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:40:57,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:40:57,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:40:57,280 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:40:58,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:03,336 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:07,915 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:41:07,917 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:41:07,917 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:41:07,917 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:41:08,036 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:41:08,956 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:10,037 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:41:12,037 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:41:12,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:41:12,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:41:12,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:41:14,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:19,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:23,419 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:41:24,420 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:27,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:41:27,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:41:27,280 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:41:27,790 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:41:27,791 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:41:27,792 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:41:27,792 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:41:28,043 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:41:29,044 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:41:29,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:33,046 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:41:34,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:39,985 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:42,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:41:42,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:41:42,280 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:41:45,398 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:47,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:41:47,093 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:41:47,093 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:41:47,094 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:41:48,052 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:41:49,052 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:41:51,198 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:53,054 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:41:53,419 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:41:56,420 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:41:57,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:41:57,279 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:41:57,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:42:02,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:42:02,014 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:42:02,014 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:42:02,015 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:02,015 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:42:02,057 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:42:05,058 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:42:07,059 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:42:07,176 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:12,177 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:12,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:42:12,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:42:12,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:42:17,323 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:17,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:42:17,393 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:42:17,393 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:42:17,394 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:42:18,063 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:42:19,063 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:42:22,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:23,065 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:42:23,420 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:42:27,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:42:27,284 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:42:27,284 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:42:28,326 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:33,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:36,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:42:36,110 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:42:36,110 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:42:36,111 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:42:37,070 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:42:39,071 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:42:39,151 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:41,072 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:42:42,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:42:42,284 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:42:42,286 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:42:44,321 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:49,322 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:52,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:42:54,323 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:42:55,507 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:42:55,507 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:42:55,510 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:42:55,510 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:42:56,078 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:42:57,078 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:42:57,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:42:57,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:42:57,284 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:42:59,079 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:42:59,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:04,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:09,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:12,282 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:43:12,283 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:43:12,323 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:43:12,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:43:12,825 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:43:12,826 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:43:12,826 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:43:13,085 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:43:14,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:15,086 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:43:17,087 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:43:19,088 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:43:20,004 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:23,422 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:43:25,423 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:27,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:43:27,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:43:27,284 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:43:30,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:43:30,046 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:43:30,047 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:43:30,049 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:43:30,092 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:43:31,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:33,093 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:43:35,094 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:43:36,228 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:41,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:42,284 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:43:42,284 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:43:42,285 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:43:46,375 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:51,375 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:52,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:43:52,276 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:43:52,277 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:43:52,277 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:43:53,100 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:43:53,422 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:43:55,101 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:43:56,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:43:57,102 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:43:57,286 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:43:57,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:43:57,287 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:44:02,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:07,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:08,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:44:08,707 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:44:08,707 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:44:08,710 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:44:09,106 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:44:11,107 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:44:12,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:44:12,287 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:44:12,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:44:12,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:13,107 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:44:17,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:22,388 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:23,423 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:44:25,203 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:44:25,204 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:44:25,204 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:44:25,204 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:44:26,112 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:44:27,113 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:44:27,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:44:27,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:44:27,807 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:44:27,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:31,114 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:44:33,341 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:38,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:42,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:44:42,671 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:44:42,671 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:44:42,673 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:44:42,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:44:42,805 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:44:42,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:44:43,118 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:44:43,924 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:45,119 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:44:47,120 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:44:49,825 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:53,424 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:44:55,425 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:44:57,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:44:57,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:44:57,806 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:45:00,893 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:02,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:45:02,081 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:45:02,081 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:45:02,081 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:45:02,125 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:45:03,126 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:45:06,208 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:07,127 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:45:11,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:12,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:45:12,807 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:45:12,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:45:16,928 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:21,929 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:23,425 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:45:25,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:45:25,457 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:45:25,457 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:45:25,458 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:45:26,134 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:45:27,135 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:45:27,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:28,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:45:28,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:45:28,093 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:45:31,136 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:45:32,637 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:37,637 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:42,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:43,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:45:43,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:45:43,091 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:45:46,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:45:46,088 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:45:46,088 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:45:46,089 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:45:46,143 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:45:47,143 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:45:48,128 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:51,145 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:45:53,312 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:45:53,426 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:45:58,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:45:58,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:45:58,090 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:45:59,201 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:04,202 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:09,203 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:10,990 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:46:10,992 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:46:10,992 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:46:10,992 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:46:11,151 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:46:13,152 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:46:13,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:46:13,582 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:46:13,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:46:15,140 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:15,153 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:46:20,141 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:23,427 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:46:25,187 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:46:25,188 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:46:25,188 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:46:25,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:25,189 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:46:26,157 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:46:27,157 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:46:28,592 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:46:28,592 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:46:28,592 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:46:29,158 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:46:30,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:31,159 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:46:35,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:40,723 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:43,583 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:46:43,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:46:43,585 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:46:44,207 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:46:44,209 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:46:44,209 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:46:44,209 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:46:45,165 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:46:45,165 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:46:46,248 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:49,166 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:46:51,417 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:53,428 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:46:56,429 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:46:58,580 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:46:58,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:46:58,582 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:47:00,426 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:47:00,427 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:47:00,428 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:47:00,428 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:47:01,170 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:47:01,467 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:03,171 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:47:05,172 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:47:06,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:11,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:13,580 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:47:13,581 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:47:13,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:47:16,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:17,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:47:17,680 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:47:17,680 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:47:17,681 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:47:18,176 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:47:19,177 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:47:21,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:23,178 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:47:23,429 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:47:27,430 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:28,580 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:47:28,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:47:28,582 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:47:32,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:35,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:47:35,720 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:47:35,721 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:47:35,721 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:47:36,183 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:47:37,183 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:47:37,760 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:41,185 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:47:43,059 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:43,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:47:43,581 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:47:43,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:47:48,700 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:53,701 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:47:55,387 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:47:58,580 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:47:58,581 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:47:58,621 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:47:59,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:00,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:48:00,242 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:48:00,242 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:48:00,242 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:48:01,193 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:48:01,193 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:48:05,194 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:48:05,394 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:10,395 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:13,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:48:13,581 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:48:13,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:48:15,717 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:17,897 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:48:17,898 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:48:17,899 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:48:17,899 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:48:18,199 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:48:19,199 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:48:20,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:23,201 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:48:23,430 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:48:26,431 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:28,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:48:28,581 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:48:28,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:48:31,675 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:31,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:48:31,943 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:48:31,943 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:48:31,945 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:48:32,204 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:48:33,204 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:48:37,094 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:37,205 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:48:42,095 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:43,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:48:43,582 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:48:43,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:48:45,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:48:45,815 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:48:45,815 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:48:45,816 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:48:46,209 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:48:47,209 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:48:47,856 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:51,211 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:48:52,975 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:53,431 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:48:58,432 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:48:58,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:48:58,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:48:58,582 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:49:03,649 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:08,339 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:49:08,340 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:49:08,340 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:49:08,342 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:49:09,217 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:49:09,217 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:49:09,380 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:13,219 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:49:13,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:49:13,582 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:49:13,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:49:14,632 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:19,633 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:23,432 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:49:25,433 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:28,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:49:28,582 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:49:28,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:49:28,802 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:49:28,803 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:49:28,803 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:49:28,804 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:49:29,224 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:49:30,843 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:31,225 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:49:33,225 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:49:35,976 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:40,977 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:43,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:49:43,582 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:49:43,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:49:45,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:49:45,351 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:49:45,352 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:49:45,354 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:49:46,232 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:49:46,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:47,233 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:49:49,234 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:49:51,489 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:53,433 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:49:57,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:49:58,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:49:58,583 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:49:58,583 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:50:02,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:05,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:50:05,630 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:50:05,631 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:50:05,631 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:50:06,239 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:50:07,240 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:50:07,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:11,241 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:50:12,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:13,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:50:13,583 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:50:13,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:50:18,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:23,434 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:50:24,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:27,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:50:27,546 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:50:27,546 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:50:27,546 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:50:28,247 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:50:29,248 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:50:29,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:50:29,274 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:50:29,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:50:30,321 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:31,249 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:50:33,249 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:50:35,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:40,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:44,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:50:44,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:50:44,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:50:46,301 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:47,776 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:50:47,777 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:50:47,778 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:50:47,778 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:50:48,255 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:50:49,256 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:50:51,977 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:53,257 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:50:53,434 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:50:57,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:50:59,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:50:59,274 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:50:59,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:51:03,310 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:08,311 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:11,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:51:11,189 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:51:11,190 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:51:11,190 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:51:11,266 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:51:13,267 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:51:14,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:14,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:51:14,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:51:14,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:51:15,268 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:51:19,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:23,435 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:51:24,436 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:29,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:51:29,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:51:29,274 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:51:30,357 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:30,377 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:51:30,379 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:51:30,379 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:51:30,379 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:51:31,278 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:51:31,279 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:51:35,280 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:51:35,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:40,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:44,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:51:44,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:51:44,276 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:51:45,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:51:45,524 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:51:45,525 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:51:45,525 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:51:46,284 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:51:46,564 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:47,285 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:51:51,286 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:51:51,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:53,436 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:51:57,438 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:51:59,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:51:59,274 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:51:59,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:52:00,794 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:52:00,796 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:52:00,796 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:52:00,797 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:52:01,290 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:52:02,836 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:03,291 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:52:05,295 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:52:08,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:13,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:14,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:52:14,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:52:14,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:52:15,100 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:52:15,101 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:52:15,101 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:52:15,102 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:52:15,299 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:52:17,300 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:52:18,141 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:19,301 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:52:23,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:23,437 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:52:28,439 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:29,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:52:29,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:52:29,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:52:34,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:36,163 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:52:36,164 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:52:36,164 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:52:36,164 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:52:36,308 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:52:37,308 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:52:39,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:41,310 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:52:44,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:52:44,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:52:44,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:52:44,398 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:49,399 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:51,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:52:54,356 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:52:54,356 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:52:54,359 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:52:54,359 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:52:54,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:52:55,315 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:52:55,315 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:52:59,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:52:59,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:52:59,275 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:53:00,378 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:05,379 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:10,380 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:12,003 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:53:12,005 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:53:12,005 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:53:12,005 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:53:12,322 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:53:13,322 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:53:14,620 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:53:14,621 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:53:14,621 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:53:16,192 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:17,324 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:53:21,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:23,439 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:53:26,440 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:29,620 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:53:29,621 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:53:29,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:53:31,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:35,474 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:53:35,475 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:53:35,475 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:53:35,477 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:53:36,330 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:53:37,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:53:37,515 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:39,332 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:53:42,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:44,619 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:53:44,619 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:53:44,620 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:53:47,697 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:50,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:53:50,459 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:53:50,460 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:53:50,460 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:53:51,336 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:53:51,336 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:53:53,440 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:53:53,441 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:55,337 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:53:58,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:53:59,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:53:59,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:53:59,644 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:54:03,775 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:07,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:54:07,353 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:54:07,353 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:54:07,356 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:54:08,342 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:54:09,342 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:54:09,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:11,343 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:54:14,513 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:14,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:54:14,629 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:54:14,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:54:19,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:22,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:54:22,647 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:54:22,647 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:54:22,648 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:54:23,348 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:54:23,348 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:54:23,441 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:54:25,442 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:27,349 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:54:29,624 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:54:29,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:54:29,625 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:54:30,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:35,710 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:40,711 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:44,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:54:44,553 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:54:44,553 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:54:44,556 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:54:44,624 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:54:44,625 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:54:44,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:54:45,356 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:54:45,356 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:54:45,746 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:49,358 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:54:50,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:53,442 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:54:56,443 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:54:59,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:54:59,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:54:59,628 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:55:01,724 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:02,728 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:55:02,729 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:55:02,729 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:55:02,730 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:55:03,363 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:55:03,363 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:55:06,885 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:07,365 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:55:11,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:14,624 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:55:14,625 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:55:14,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:55:17,682 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:20,138 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:55:20,139 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:55:20,140 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:55:20,141 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:55:20,369 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:55:21,370 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:55:23,151 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:23,443 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:55:25,371 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:55:28,541 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:29,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:55:29,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:55:29,627 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:55:33,755 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:38,756 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:42,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:55:42,914 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:55:42,914 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:55:42,915 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:55:43,378 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:55:43,378 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:55:43,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:44,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:55:44,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:55:44,627 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:55:47,379 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:55:49,028 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:53,444 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:55:54,445 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:59,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:55:59,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:55:59,626 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:55:59,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:56:00,712 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:56:00,713 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:56:00,713 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:56:00,713 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:56:01,384 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:56:01,385 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:56:04,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:05,385 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:56:09,832 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:14,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:56:14,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:56:14,627 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:56:15,744 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:19,144 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:56:19,145 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:56:19,145 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:56:19,146 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:56:19,391 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:56:19,391 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:56:21,185 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:23,392 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:56:23,445 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:56:26,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:29,844 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:56:29,844 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:56:29,845 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:56:31,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:36,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:56:36,393 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:56:36,393 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:56:36,394 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:56:36,397 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:56:37,397 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:56:37,433 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:41,399 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:56:42,572 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:44,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:56:44,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:56:44,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:56:47,885 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:52,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:53,446 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:56:56,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:56:56,842 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:56:56,843 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:56:56,843 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:56:57,405 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:56:57,405 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:56:58,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:56:59,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:56:59,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:56:59,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:57:01,406 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:57:03,972 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:08,973 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:13,973 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:14,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:57:14,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:57:14,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:57:19,897 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:21,104 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:57:21,105 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:57:21,106 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:57:21,106 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:57:21,413 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:57:21,414 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:57:23,447 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:57:25,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:25,415 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:57:29,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:57:29,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:57:29,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:57:30,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:35,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:38,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:57:38,009 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:57:38,010 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:57:38,010 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:57:38,419 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:57:39,420 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:57:41,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:41,420 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:57:44,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:57:44,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:57:44,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:57:46,915 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:51,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:53,452 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:57:57,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:57:59,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:57:59,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:57:59,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:58:01,264 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:58:01,265 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:58:01,265 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:58:01,265 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:58:01,428 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:58:01,428 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:58:03,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:05,430 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:58:08,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:13,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:14,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:58:14,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:58:14,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:58:18,764 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:58:18,765 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:58:18,766 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:58:18,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:18,767 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:58:19,435 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:58:19,435 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:58:23,437 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:58:23,449 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:58:24,450 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:29,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:29,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:58:29,831 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:58:29,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:58:35,113 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:35,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:58:35,717 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:58:35,717 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:58:35,719 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:58:36,441 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:58:37,441 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:58:39,444 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:58:40,832 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:44,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:58:44,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:58:44,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:58:45,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:50,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:52,923 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:58:52,924 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:58:52,925 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:58:52,925 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:58:53,449 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:58:53,450 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:58:54,450 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:58:55,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:58:58,451 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:58:59,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:58:59,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:58:59,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:59:01,922 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:06,923 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:09,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:59:09,553 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:59:09,553 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:59:09,555 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:59:10,456 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:59:12,457 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:59:12,592 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:14,457 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:59:14,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:59:14,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:59:14,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:59:17,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:22,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:23,451 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:59:24,288 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:59:24,289 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:59:24,290 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:59:24,290 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:59:24,461 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:59:26,462 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:59:28,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:28,463 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:59:29,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:59:29,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:59:29,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:59:33,899 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:38,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:43,901 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:44,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:59:44,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:59:44,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 04:59:44,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 04:59:44,928 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 04:59:44,928 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 04:59:44,931 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 04:59:45,470 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 04:59:46,470 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:59:49,082 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:50,472 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 04:59:53,452 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 04:59:54,453 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:59,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 04:59:59,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 04:59:59,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 04:59:59,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:00:03,225 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:00:03,227 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:00:03,227 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:00:03,227 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:00:03,477 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:00:04,477 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:00:05,267 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:08,479 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:00:10,377 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:14,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:00:14,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:00:14,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:00:15,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:20,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:23,042 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:00:23,045 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:00:23,045 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:00:23,048 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:00:23,453 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:00:23,485 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:00:24,486 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:00:26,059 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:28,487 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:00:29,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:00:29,831 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:00:29,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:00:31,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:36,932 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:41,933 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:43,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:00:43,247 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:00:43,248 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:00:43,248 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:00:43,493 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:00:44,493 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:00:44,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:00:44,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:00:44,961 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:00:47,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:48,495 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:00:52,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:53,455 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:00:57,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:00:59,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:00:59,959 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:00:59,961 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:01:00,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:01:00,640 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:01:00,640 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:01:00,641 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:01:01,500 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:01:02,500 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:01:02,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:06,502 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:01:07,836 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:12,837 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:14,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:01:14,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:01:14,961 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:01:17,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:01:17,328 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:01:17,328 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:01:17,329 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:01:17,506 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:01:18,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:18,507 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:01:22,508 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:01:23,456 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:01:23,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:28,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:29,959 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:01:29,959 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:01:29,959 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:01:33,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:34,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:01:34,859 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:01:34,859 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:01:34,860 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:01:35,514 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:01:36,514 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:01:39,975 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:40,516 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:01:44,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:01:44,960 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:01:44,963 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:01:45,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:50,028 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:53,457 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:01:55,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:01:59,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:01:59,961 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:01:59,961 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:02:01,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:01,727 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:02:01,728 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:02:01,728 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:02:01,728 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:02:02,523 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:02:04,524 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:02:06,525 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:02:06,868 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:11,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:14,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:02:14,960 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:02:14,963 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:02:17,017 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:22,017 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:23,122 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:02:23,123 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:02:23,123 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:02:23,124 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:02:23,457 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:02:23,531 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:02:24,532 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:02:27,288 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:28,533 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:02:29,959 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:02:29,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:02:29,961 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:02:33,093 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:38,094 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:42,061 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:02:42,062 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:02:42,063 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:02:42,063 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:02:42,538 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:02:43,102 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:44,539 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:02:44,979 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:02:44,981 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:02:44,982 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:02:46,540 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:02:48,235 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:53,235 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:53,459 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:02:58,460 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:02:59,968 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:02:59,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:02:59,969 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:03:02,178 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:03:02,179 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:03:02,179 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:03:02,180 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:03:02,546 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:03:04,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:04,547 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:03:06,547 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:03:08,548 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:03:09,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:14,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:14,967 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:03:14,968 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:03:14,968 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:03:19,988 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:20,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:03:20,561 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:03:20,561 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:03:20,562 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:03:21,553 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:03:22,553 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:03:23,460 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:03:25,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:26,555 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:03:29,968 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:03:29,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:03:29,969 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:03:31,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:36,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:38,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:03:38,291 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:03:38,291 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:03:38,291 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:03:38,559 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:03:40,560 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:03:41,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:42,561 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:03:44,562 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:03:44,995 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:03:44,996 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:03:44,996 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:03:47,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:52,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:53,460 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:03:57,461 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:03:58,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:03:58,059 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:03:58,059 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:03:58,060 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:03:58,568 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:04:00,569 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:04:00,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:04:00,653 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:04:00,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:04:02,570 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:04:03,192 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:08,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:13,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:15,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:04:15,316 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:04:15,317 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:04:15,317 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:04:15,575 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:04:15,649 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:04:15,650 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:04:15,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:04:16,575 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:04:18,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:20,577 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:04:23,461 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:04:23,464 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:28,462 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:30,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:04:30,650 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:04:30,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:04:33,705 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:38,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:41,061 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:04:41,062 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:04:41,062 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:04:41,063 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:04:41,584 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:04:42,584 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:04:44,061 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:45,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:04:45,653 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:04:45,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:04:46,586 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:04:49,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:53,462 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:04:55,463 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:04:55,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:04:55,781 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:04:55,781 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:04:55,781 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:04:56,589 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:04:58,590 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:05:00,591 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:05:00,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:05:00,650 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:05:00,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:05:00,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:05,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:10,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:15,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:05:15,650 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:05:15,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:05:15,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:17,259 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:05:17,261 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:05:17,261 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:05:17,261 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:05:17,597 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:05:20,598 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:05:21,687 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:22,599 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:05:23,463 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:05:27,464 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:30,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:05:30,651 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:05:30,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:05:32,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:37,770 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:41,339 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:05:41,340 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:05:41,340 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:05:41,341 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:05:41,606 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:05:42,607 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:05:43,380 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:45,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:05:45,651 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:05:45,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:05:46,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:05:48,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:53,464 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:05:54,465 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:05:59,466 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:00,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:06:00,615 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:06:00,615 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:06:00,616 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:06:00,616 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:06:00,655 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:06:00,655 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:06:00,656 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:06:02,617 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:06:05,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:06,619 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:06:10,015 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:15,016 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:15,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:06:15,651 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:06:15,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:06:20,781 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:22,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:06:22,657 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:06:22,657 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:06:22,657 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:06:23,465 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:06:23,625 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:06:24,626 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:06:25,840 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:28,628 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:06:30,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:06:30,651 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:06:30,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:06:31,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:36,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:39,450 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:06:39,451 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:06:39,451 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:06:39,452 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:06:39,632 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:06:40,632 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:06:42,443 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:44,634 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:06:45,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:06:45,651 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:06:45,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:06:47,751 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:52,752 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:53,467 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:06:57,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:06:57,310 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:06:57,310 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:06:57,311 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:06:57,639 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:06:58,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:06:58,640 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:07:00,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:07:00,651 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:07:00,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:07:02,648 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:07:03,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:08,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:13,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:15,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:07:15,651 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:07:15,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:07:16,884 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:07:16,886 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:07:16,886 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:07:16,886 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:07:17,656 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:07:18,657 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:07:18,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:22,658 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:07:23,467 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:07:24,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:29,469 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:30,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:07:30,652 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:07:30,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:07:34,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:39,781 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:40,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:07:40,780 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:07:40,781 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:07:40,781 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:07:41,665 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:07:42,665 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:07:44,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:45,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:07:45,653 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:07:45,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:07:46,667 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:07:50,801 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:53,469 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:07:56,470 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:07:57,404 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:07:57,405 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:07:57,406 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:07:57,406 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:07:57,671 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:07:58,671 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:08:00,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:08:00,652 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:08:00,655 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:08:01,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:02,673 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:08:06,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:11,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:12,996 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:08:12,997 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:08:12,997 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:08:12,997 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:08:13,677 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:08:14,677 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:08:15,712 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:08:15,713 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:08:15,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:08:17,143 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:18,679 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:08:22,143 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:23,469 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:08:27,470 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:29,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:08:29,908 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:08:29,909 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:08:29,909 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:08:30,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:08:30,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:08:30,660 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:08:30,684 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:08:32,684 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:08:32,752 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:34,685 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:08:38,059 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:43,060 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:45,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:08:45,659 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:08:45,700 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:08:48,005 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:08:48,006 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:08:48,006 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:08:48,007 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:08:48,691 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:08:49,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:50,691 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:08:52,692 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:08:53,470 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:08:54,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:08:59,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:00,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:09:00,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:09:00,659 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:09:04,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:08,474 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:09:08,475 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:09:08,475 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:09:08,476 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:09:08,698 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:09:10,515 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:10,699 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:09:12,700 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:09:15,635 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:15,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:09:15,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:09:15,660 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:09:20,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:23,471 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:09:26,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:09:26,346 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:09:26,346 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:09:26,347 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:26,347 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:09:26,705 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:09:28,706 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:09:30,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:09:30,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:09:30,661 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:09:30,707 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:09:31,725 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:36,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:41,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:45,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:09:45,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:09:45,660 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:09:46,699 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:09:46,700 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:09:46,700 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:09:46,701 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:09:46,712 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:09:46,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:48,713 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:09:50,714 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:09:51,810 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:09:53,472 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:09:57,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:00,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:10:00,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:10:00,660 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:10:00,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:10:00,873 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:10:00,874 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:10:00,874 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:10:01,718 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:10:02,718 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:10:02,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:04,719 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:10:06,720 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:10:08,000 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:13,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:15,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:10:15,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:10:15,661 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:10:18,729 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:19,267 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:10:19,268 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:10:19,268 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:10:19,269 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:10:19,725 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:10:20,725 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:10:23,473 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:10:24,474 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:24,727 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:10:29,475 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:30,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:10:30,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:10:30,660 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:10:34,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:39,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:41,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:10:41,161 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:10:41,162 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:10:41,162 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:10:41,734 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:10:42,734 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:10:45,418 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:45,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:10:45,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:10:45,664 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:10:46,736 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:10:50,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:10:53,474 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:10:56,475 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:00,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:11:00,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:11:00,660 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:11:01,787 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:02,517 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:11:02,518 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:11:02,518 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:11:02,519 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:11:02,741 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:11:04,742 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:11:06,743 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:11:07,701 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:12,702 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:15,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:11:15,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:11:15,661 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:11:17,711 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:19,260 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:11:19,261 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:11:19,261 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:11:19,262 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:11:19,748 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:11:20,749 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:11:23,405 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:23,475 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:11:24,750 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:11:28,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:30,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:11:30,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:11:30,661 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:11:33,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:38,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:39,140 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:11:39,141 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:11:39,142 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:11:39,142 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:11:39,756 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:11:40,756 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:11:44,375 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:44,758 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:11:45,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:11:45,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:11:45,661 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:11:49,690 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:53,476 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:11:55,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:11:57,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:11:57,867 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:11:57,867 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:11:57,867 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:11:58,763 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:11:58,763 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:12:00,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:12:00,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:00,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:12:00,661 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:12:02,764 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:12:06,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:11,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:15,184 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:12:15,185 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:12:15,186 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:12:15,186 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:12:15,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:12:15,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:12:15,662 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:12:15,769 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:12:16,735 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:16,770 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:12:20,771 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:12:22,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:23,477 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:12:27,478 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:30,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:12:30,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:12:30,662 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:12:32,166 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:12:32,167 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:12:32,167 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:12:32,168 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:12:32,777 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:12:33,198 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:34,778 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:12:36,779 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:12:38,329 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:43,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:45,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:12:45,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:12:45,661 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:12:48,521 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:12:48,522 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:12:48,522 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:12:48,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:48,523 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:12:48,783 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:12:50,784 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:12:52,785 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:12:53,478 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:12:54,479 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:12:59,479 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:00,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:13:00,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:13:00,661 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:13:03,255 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:13:03,256 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:13:03,256 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:13:03,257 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:13:03,789 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:13:04,789 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:13:05,295 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:08,791 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:13:11,179 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:15,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:13:15,665 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:13:15,665 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:13:16,792 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:21,793 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:23,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:13:23,206 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:13:23,206 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:13:23,207 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:13:23,479 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:13:23,796 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:13:24,797 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:13:27,630 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:28,798 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:13:30,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:13:30,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:13:30,665 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:13:32,784 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:37,785 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:42,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:13:42,737 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:13:42,737 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:13:42,738 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:13:42,803 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:13:43,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:44,804 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:13:45,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:13:45,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:13:45,662 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:13:46,805 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:13:48,928 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:53,480 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:13:54,481 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:13:59,481 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:00,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:14:00,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:14:00,663 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:14:04,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:06,162 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:14:06,163 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:14:06,163 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:14:06,164 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:14:06,812 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:14:06,812 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:14:10,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:10,813 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:14:15,310 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:15,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:14:15,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:14:15,663 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:14:20,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:14:20,615 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:14:20,615 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:14:20,616 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:14:20,616 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:20,817 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:14:22,818 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:14:23,481 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:14:24,819 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:14:25,789 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:30,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:14:30,665 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:14:30,666 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:14:31,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:36,710 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:39,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:14:39,346 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:14:39,347 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:14:39,347 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:14:39,824 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:14:40,825 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:14:42,371 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:44,830 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:14:45,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:14:45,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:14:45,664 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:14:47,760 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:52,761 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:53,482 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:14:55,199 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:14:55,200 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:14:55,201 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:14:55,201 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:14:55,834 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:14:56,835 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:14:58,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:14:58,836 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:15:00,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:15:00,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:15:00,663 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:15:03,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:08,755 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:11,515 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:15:11,516 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:15:11,516 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:15:11,517 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:15:11,840 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:15:12,840 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:15:14,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:15,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:15:15,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:15:15,664 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:15:16,842 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:15:19,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:23,483 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:15:25,483 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:26,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:15:26,344 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:15:26,344 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:15:26,344 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:15:26,847 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:15:26,847 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:15:30,519 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:30,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:15:30,710 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:15:30,710 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:15:30,848 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:15:35,802 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:40,803 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:45,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:15:45,668 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:15:45,669 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:15:46,772 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:51,773 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:15:52,329 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:15:52,330 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:15:52,330 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:15:52,331 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:15:52,857 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:15:52,858 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:15:53,483 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:15:56,859 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:15:57,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:00,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:16:00,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:16:00,670 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:16:02,790 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:07,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:10,181 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:16:10,182 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:16:10,182 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:16:10,183 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:16:10,865 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:16:10,865 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:16:13,197 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:14,866 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:16:15,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:16:15,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:16:15,669 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:16:18,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:23,484 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:16:24,485 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:29,486 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:30,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:16:30,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:16:30,669 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:16:32,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:16:32,689 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:16:32,690 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:16:32,690 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:16:32,873 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:16:34,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:34,874 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:16:36,875 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:16:39,839 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:44,840 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:45,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:16:45,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:16:45,669 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:16:50,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:16:50,392 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:16:50,392 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:16:50,393 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:50,393 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:16:50,881 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:16:51,881 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:16:53,485 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:16:55,529 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:16:55,883 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:17:00,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:00,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:17:00,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:17:00,669 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:17:04,148 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:17:04,150 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:17:04,150 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:17:04,151 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:17:04,887 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:17:05,887 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:17:06,184 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:09,889 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:17:11,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:15,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:17:15,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:17:15,670 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:17:16,788 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:21,789 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:22,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:17:22,864 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:17:22,865 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:17:22,865 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:17:22,894 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:17:23,486 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:17:25,895 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:17:27,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:27,896 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:17:30,674 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:17:30,675 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:17:30,676 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:17:32,768 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:37,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:40,173 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:17:40,175 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:17:40,175 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:17:40,176 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:17:40,900 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:17:41,901 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:17:43,189 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:45,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:17:45,674 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:17:45,674 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:17:45,902 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:17:48,778 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:53,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:54,262 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:17:56,067 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:17:56,068 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:17:56,068 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:17:56,071 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:17:56,908 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:17:57,908 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:17:59,453 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:17:59,909 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:18:00,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:18:00,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:18:00,681 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:18:04,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:09,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:13,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:18:13,746 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:18:13,746 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:18:13,747 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:18:13,915 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:18:15,777 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:15,915 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:18:16,382 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:18:16,383 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:18:16,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:18:19,917 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:18:20,929 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:23,488 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:18:26,489 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:31,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:18:31,384 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:18:31,388 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:18:31,508 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:36,056 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:18:36,058 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:18:36,058 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:18:36,060 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:18:36,923 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:18:37,097 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:37,924 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:18:41,926 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:18:42,555 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:46,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:18:46,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:18:46,386 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:18:48,483 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:53,484 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:53,489 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:18:53,665 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:18:53,666 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:18:53,666 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:18:53,667 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:18:53,930 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:18:55,931 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:18:58,847 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:18:59,933 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:19:01,382 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:19:01,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:19:01,383 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:19:04,436 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:09,437 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:10,152 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:19:10,153 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:19:10,154 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:19:10,156 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:19:10,938 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:19:11,938 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:19:15,425 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:15,940 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:19:16,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:19:16,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:19:16,385 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:19:21,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:23,490 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:19:23,790 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:19:23,792 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:19:23,792 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:19:23,792 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:19:23,943 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:19:25,945 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:19:26,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:29,946 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:19:31,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:19:31,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:19:31,385 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:19:32,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:37,469 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:41,196 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:19:41,198 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:19:41,198 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:19:41,200 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:19:41,951 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:19:43,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:43,952 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:19:45,953 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:19:46,382 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:19:46,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:19:46,384 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:19:48,460 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:53,461 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:53,490 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:19:58,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:19:59,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:19:59,599 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:19:59,599 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:19:59,599 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:19:59,958 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:20:01,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:20:01,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:20:01,383 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:20:01,959 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:03,857 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:03,960 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:05,961 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:08,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:13,859 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:16,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:20:16,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:20:16,384 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:20:18,095 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:20:18,096 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:20:18,097 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:20:18,099 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:20:18,965 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:20:19,128 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:19,966 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:23,491 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:20:23,967 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:24,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:29,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:31,382 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:20:31,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:20:31,384 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:20:32,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:20:32,902 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:20:32,902 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:20:32,902 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:20:32,971 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:20:34,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:35,972 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:37,973 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:40,068 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:45,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:46,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:20:46,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:20:46,384 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:20:49,396 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:20:49,397 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:20:49,397 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:20:49,399 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:20:49,978 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:20:50,437 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:20:51,978 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:53,492 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:20:53,979 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:20:55,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:00,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:01,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:21:01,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:21:01,384 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:21:06,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:11,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:13,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:21:13,832 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:21:13,832 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:21:13,832 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:21:13,987 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:21:15,988 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:21:16,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:21:16,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:21:16,450 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:21:16,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:17,989 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:21:19,989 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:21:22,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:23,493 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:21:27,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:31,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:21:31,448 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:21:31,488 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:21:32,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:33,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:21:33,781 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:21:33,781 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:21:33,782 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:21:33,995 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:21:35,996 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:21:37,996 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:21:38,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:39,997 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:21:43,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:46,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:21:46,447 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:21:46,488 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:21:48,550 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:53,494 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:21:54,495 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:21:59,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:01,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:22:01,449 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:22:01,489 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:22:01,764 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:22:01,765 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:22:01,766 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:22:01,766 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:22:02,005 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:22:04,006 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:22:04,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:06,007 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:22:08,007 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:22:10,204 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:15,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:16,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:22:16,450 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:22:16,450 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:22:20,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:23,495 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:22:23,827 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:22:23,829 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:22:23,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:22:23,829 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:22:24,013 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:22:25,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:26,014 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:22:28,015 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:22:30,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:31,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:22:31,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:22:31,449 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:22:36,477 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:40,399 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:22:40,400 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:22:40,401 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:22:40,401 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:22:41,020 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:22:42,020 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:22:42,440 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:46,022 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:22:46,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:22:46,450 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:22:46,490 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:22:47,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:52,575 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:22:53,502 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:22:58,503 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:01,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:23:01,450 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:23:01,490 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:23:03,509 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:05,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:23:05,904 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:23:05,904 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:23:05,905 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:23:06,029 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:23:08,030 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:23:08,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:10,031 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:23:12,032 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:23:14,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:16,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:23:16,449 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:23:16,490 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:23:19,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:21,264 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:23:21,266 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:23:21,266 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:23:21,267 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:23:22,036 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:23:23,498 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:23:24,036 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:23:24,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:26,037 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:23:29,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:31,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:23:31,451 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:23:31,491 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:23:35,542 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:40,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:44,774 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:23:44,775 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:23:44,775 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:23:44,776 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:23:45,044 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:23:45,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:46,044 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:23:46,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:23:46,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:23:46,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:23:50,046 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:23:51,196 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:23:53,498 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:23:56,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:01,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:24:01,493 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:24:01,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:24:01,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:03,227 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:24:03,228 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:24:03,228 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:24:03,230 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:24:04,051 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:24:06,052 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:24:07,429 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:08,053 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:24:12,429 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:16,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:24:16,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:24:16,493 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:24:17,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:22,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:23,500 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:24:25,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:24:25,305 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:24:25,306 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:24:25,306 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:24:26,059 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:24:28,060 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:24:28,346 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:30,061 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:24:31,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:24:31,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:24:31,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:24:33,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:38,561 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:42,024 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:24:42,025 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:24:42,025 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:24:42,027 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:24:42,065 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:24:44,065 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:44,066 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:24:46,067 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:24:46,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:24:46,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:24:46,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:24:49,542 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:24:53,500 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:24:55,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:00,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:01,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:25:01,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:25:01,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:25:01,842 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:25:01,844 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:25:01,844 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:25:01,844 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:25:02,073 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:25:04,074 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:25:06,021 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:06,075 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:25:11,022 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:16,023 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:16,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:25:16,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:25:16,497 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:25:21,536 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:23,502 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:25:25,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:25:25,359 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:25:25,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:25:25,360 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:25:26,081 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:25:27,399 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:28,082 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:25:30,083 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:25:31,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:25:31,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:25:31,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:25:32,575 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:37,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:42,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:46,495 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:25:46,496 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:25:46,536 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:25:48,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:48,792 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:25:48,793 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:25:48,793 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:25:48,794 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:25:49,090 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:25:50,090 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:25:53,503 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:25:54,092 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:25:54,504 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:25:59,505 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:01,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:26:01,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:26:01,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:26:04,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:09,154 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:26:09,155 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:26:09,155 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:26:09,155 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:26:10,098 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:26:10,098 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:26:10,195 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:14,099 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:26:15,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:16,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:26:16,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:26:16,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:26:20,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:23,504 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:26:25,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:26:25,237 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:26:25,238 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:26:25,238 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:26:26,104 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:26:26,105 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:26:26,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:30,106 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:26:31,372 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:31,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:26:31,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:26:31,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:26:36,608 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:41,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:46,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:26:46,495 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:26:46,535 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:26:47,572 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:51,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:26:51,320 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:26:51,321 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:26:51,321 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:26:52,115 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:26:52,115 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:26:53,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:26:53,505 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:26:56,116 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:26:58,478 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:01,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:27:01,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:27:01,539 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:27:03,590 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:07,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:27:07,608 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:27:07,609 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:27:07,609 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:27:08,121 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:27:08,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:10,122 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:27:12,123 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:27:13,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:16,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:27:16,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:27:16,508 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:27:19,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:23,505 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:27:25,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:29,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:27:29,472 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:27:29,473 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:27:29,473 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:27:30,130 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:27:30,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:32,068 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:27:32,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:27:32,069 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:27:32,131 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:27:34,132 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:27:35,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:40,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:44,846 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:27:44,847 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:27:44,847 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:27:44,848 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:27:45,136 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:27:45,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:46,136 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:27:47,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:27:47,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:27:47,472 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:27:50,138 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:27:51,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:27:53,507 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:27:56,508 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:01,509 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:02,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:28:02,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:28:02,473 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:28:02,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:28:02,831 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:28:02,832 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:28:02,832 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:28:03,142 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:28:04,143 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:28:07,012 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:08,144 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:28:12,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:15,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:28:15,883 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:28:15,883 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:28:15,883 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:28:16,148 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:28:17,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:28:17,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:28:17,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:17,588 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:28:18,149 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:28:20,150 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:28:23,113 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:23,508 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:28:28,509 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:32,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:28:32,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:28:32,587 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:28:33,696 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:37,469 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:28:37,470 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:28:37,470 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:28:37,471 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:28:38,156 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:28:38,157 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:28:39,510 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:42,158 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:28:44,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:47,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:28:47,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:28:47,588 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:28:50,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:28:53,509 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:28:56,510 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:01,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:02,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:29:02,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:29:02,588 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:29:06,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:29:06,108 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:29:06,108 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:29:06,109 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:29:06,167 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:29:07,148 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:08,168 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:29:10,169 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:29:12,262 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:17,263 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:17,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:29:17,587 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:29:17,590 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:29:22,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:23,510 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:29:24,293 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:29:24,294 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:29:24,295 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:29:24,295 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:29:25,174 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:29:26,174 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:29:28,175 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:29:28,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:32,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:29:32,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:29:32,588 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:29:33,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:38,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:29:38,252 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:29:38,252 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:29:38,252 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:29:39,179 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:29:39,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:40,180 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:29:42,180 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:29:44,420 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:47,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:29:47,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:29:47,588 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:29:49,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:53,510 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:29:55,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:29:55,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:29:55,932 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:29:55,932 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:29:55,933 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:29:56,185 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:29:58,186 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:30:00,187 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:30:01,104 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:02,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:30:02,589 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:30:02,589 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:30:06,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:11,437 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:30:11,439 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:30:11,439 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:30:11,439 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:30:12,191 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:30:12,192 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:30:12,478 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:16,193 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:30:17,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:17,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:30:17,622 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:30:17,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:30:22,743 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:23,512 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:30:26,876 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:30:26,878 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:30:26,878 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:30:26,878 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:30:27,197 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:30:27,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:28,198 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:30:32,199 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:30:32,593 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:30:32,593 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:30:32,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:30:33,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:38,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:43,628 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:45,578 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:30:45,579 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:30:45,579 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:30:45,579 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:30:46,205 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:30:46,205 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:30:48,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:30:48,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:30:48,240 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:30:48,657 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:50,207 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:30:53,513 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:30:54,514 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:30:59,515 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:03,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:31:03,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:31:03,278 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:31:04,259 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:31:04,260 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:31:04,261 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:31:05,253 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:31:05,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:06,213 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:31:06,213 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:31:08,214 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:31:10,408 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:15,409 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:18,245 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:31:18,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:31:18,247 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:31:19,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:31:19,033 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:31:19,033 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:31:19,033 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:31:19,218 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:31:20,219 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:31:21,071 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:23,514 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:31:24,220 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:31:26,515 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:31,516 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:33,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:31:33,243 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:31:33,245 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:31:36,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:31:36,270 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:31:36,270 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:31:36,271 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:31:37,225 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:31:37,310 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:38,226 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:31:40,226 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:31:42,450 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:47,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:48,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:31:48,243 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:31:48,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:31:53,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:31:53,514 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:31:53,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:31:53,532 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:31:53,532 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:31:53,532 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:31:54,232 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:31:54,233 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:31:58,233 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:31:58,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:03,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:32:03,243 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:32:03,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:32:04,311 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:09,311 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:10,687 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:32:10,688 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:32:10,688 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:32:10,689 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:32:11,238 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:32:12,238 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:32:14,239 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:32:14,893 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:16,240 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:32:18,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:32:18,243 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:32:18,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:32:20,306 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:23,515 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:32:25,516 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:28,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:32:28,653 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:32:28,654 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:32:28,654 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:32:29,244 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:32:30,245 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:32:30,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:32,246 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:32:33,244 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:32:33,244 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:32:33,248 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:32:36,325 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:41,326 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:46,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:46,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:32:46,830 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:32:46,831 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:32:46,831 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:32:47,254 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:32:48,254 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:32:48,538 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:32:48,539 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:32:48,539 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:32:50,255 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:32:51,933 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:32:52,256 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:32:54,707 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:32:57,708 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:02,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:03,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:33:03,539 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:33:03,539 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:33:08,573 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:10,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:33:10,498 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:33:10,499 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:33:10,499 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:33:11,263 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:33:12,263 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:33:13,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:14,264 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:33:18,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:33:18,539 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:33:18,539 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:33:19,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:23,517 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:33:25,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:30,519 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:30,926 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:33:30,927 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:33:30,928 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:33:30,929 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:33:31,271 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:33:32,271 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:33:33,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:33:33,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:33:33,657 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:33:36,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:36,273 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:33:41,111 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:46,111 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:47,517 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:33:47,518 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:33:47,518 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:33:47,519 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:33:48,277 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:33:48,278 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:33:49,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:33:49,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:33:49,222 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:33:51,676 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:33:52,279 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:33:53,518 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:33:57,519 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:02,520 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:04,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:34:04,222 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:34:04,262 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:34:06,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:34:06,499 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:34:06,499 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:34:06,501 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:34:07,284 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:34:07,538 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:08,284 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:34:10,290 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:34:12,759 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:17,760 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:19,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:34:19,221 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:34:19,261 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:34:23,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:23,519 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:34:25,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:34:25,070 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:34:25,071 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:34:25,071 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:34:25,295 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:34:27,296 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:34:29,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:31,298 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:34:34,212 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:34,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:34:34,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:34:34,221 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:34:39,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:44,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:48,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:34:48,365 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:34:48,366 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:34:48,368 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:34:49,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:34:49,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:34:49,221 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:34:49,305 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:34:49,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:51,306 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:34:53,307 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:34:53,520 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:34:54,521 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:34:59,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:04,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:35:04,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:35:04,242 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:35:05,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:35:05,240 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:35:05,240 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:35:05,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:05,241 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:35:05,312 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:35:07,312 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:35:10,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:11,314 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:35:15,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:19,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:35:19,226 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:35:19,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:35:21,357 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:23,521 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:35:25,139 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:35:25,140 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:35:25,140 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:35:25,141 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:35:25,319 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:35:27,180 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:27,320 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:35:31,321 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:35:32,441 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:34,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:35:34,227 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:35:34,232 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:35:38,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:40,690 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:35:40,691 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:35:40,691 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:35:40,691 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:35:41,325 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:35:43,326 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:35:43,732 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:45,327 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:35:49,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:49,228 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:35:49,228 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:35:49,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:35:53,522 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:35:54,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:35:57,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:35:57,600 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:35:57,601 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:35:57,601 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:35:58,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:35:59,332 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:35:59,640 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:03,334 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:36:04,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:36:04,227 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:36:04,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:36:05,272 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:10,272 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:15,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:19,227 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:36:19,227 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:36:19,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:36:20,804 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:36:20,806 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:36:20,806 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:36:20,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:20,807 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:36:21,341 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:36:23,342 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:36:23,523 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:36:25,343 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:36:26,077 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:31,078 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:34,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:36:34,227 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:36:34,232 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:36:36,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:39,038 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:36:39,040 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:36:39,040 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:36:39,040 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:36:39,348 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:36:41,349 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:36:42,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:43,350 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:36:47,195 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:49,228 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:36:49,228 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:36:49,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:36:52,311 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:53,524 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:36:54,781 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:36:54,782 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:36:54,782 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:36:54,783 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:36:55,354 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:36:57,355 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:36:57,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:36:59,356 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:37:02,974 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:04,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:37:04,230 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:37:04,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:37:08,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:13,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:14,223 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:37:14,224 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:37:14,224 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:37:14,225 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:37:14,362 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:37:15,363 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:37:18,572 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:19,227 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:37:19,228 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:37:19,232 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:37:19,364 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:37:23,525 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:37:24,525 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:29,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:31,590 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:37:31,591 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:37:31,591 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:37:31,592 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:37:32,369 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:37:33,370 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:37:34,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:37:34,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:37:34,243 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:37:34,686 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:37,372 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:37:39,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:44,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:49,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:37:49,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:37:49,243 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:37:50,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:53,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:37:53,493 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:37:53,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:37:53,494 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:37:53,526 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:37:54,378 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:37:55,379 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:37:55,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:37:59,380 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:38:00,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:04,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:38:04,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:38:04,242 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:38:06,314 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:11,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:14,187 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:38:14,188 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:38:14,188 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:38:14,188 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:38:14,386 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:38:15,386 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:38:17,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:19,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:38:19,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:38:19,244 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:38:19,388 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:38:22,347 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:23,527 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:38:27,528 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:32,529 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:32,988 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:38:32,989 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:38:32,990 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:38:32,990 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:38:33,394 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:38:34,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:38:34,722 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:38:34,763 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:38:35,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:38:37,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:38:38,406 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:39,396 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:38:43,407 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:48,408 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:49,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:38:49,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:38:49,721 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:38:53,528 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:38:53,528 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:58,529 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:38:59,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:38:59,821 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:38:59,821 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:38:59,822 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:39:00,405 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:39:01,405 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:39:03,982 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:04,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:39:04,721 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:39:04,762 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:39:05,407 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:39:09,772 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:14,773 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:19,257 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:39:19,258 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:39:19,258 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:39:19,259 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:39:19,412 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:39:19,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:39:19,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:39:19,721 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:39:20,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:21,413 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:39:23,414 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:39:23,529 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:39:26,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:31,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:34,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:39:34,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:39:34,892 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:39:36,972 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:41,562 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:39:41,564 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:39:41,564 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:39:41,564 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:39:42,422 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:39:42,596 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:43,422 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:39:47,424 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:39:47,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:49,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:39:49,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:39:49,882 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:39:52,968 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:53,529 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:39:58,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:39:59,054 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:39:59,055 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:39:59,055 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:39:59,056 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:39:59,429 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:40:01,430 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:40:03,431 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:40:04,172 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:04,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:40:04,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:40:04,882 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:40:09,966 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:14,967 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:19,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:40:19,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:40:19,882 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:40:20,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:23,531 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:40:25,532 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:27,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:40:27,345 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:40:27,345 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:40:27,346 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:40:27,440 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:40:29,441 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:40:31,442 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:40:31,484 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:34,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:40:34,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:40:34,882 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:40:36,944 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:41,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:45,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:40:45,361 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:40:45,361 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:40:45,361 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:40:45,447 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:40:47,400 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:47,448 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:40:49,449 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:40:49,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:40:49,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:40:49,882 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:40:52,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:40:53,532 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:40:58,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:02,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:41:02,070 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:41:02,070 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:41:02,070 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:41:02,454 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:41:03,454 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:41:04,101 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:04,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:41:04,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:41:04,882 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:41:07,456 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:41:09,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:14,270 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:17,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:41:17,239 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:41:17,239 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:41:17,240 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:41:17,460 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:41:19,272 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:19,461 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:41:19,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:41:19,959 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:41:19,959 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:41:21,461 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:41:23,533 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:41:24,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:29,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:34,535 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:34,883 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:41:34,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:41:34,891 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:41:39,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:43,370 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:41:43,371 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:41:43,372 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:41:43,372 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:41:43,470 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:41:45,404 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:45,471 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:41:47,472 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:41:49,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:41:49,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:41:49,891 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:41:50,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:41:53,534 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:41:56,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:01,538 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:01,787 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:42:01,788 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:42:01,789 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:42:01,789 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:42:02,477 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:42:03,478 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:42:04,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:42:04,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:42:04,892 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:42:06,990 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:07,480 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:42:11,991 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:16,991 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:18,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:42:18,888 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:42:18,888 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:42:18,888 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:42:19,484 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:42:20,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:42:20,610 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:42:20,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:42:21,485 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:42:22,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:23,486 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:42:23,534 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:42:27,536 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:32,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:35,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:42:35,610 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:42:35,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:42:37,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:39,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:42:39,341 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:42:39,341 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:42:39,342 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:42:39,492 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:42:41,493 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:42:43,493 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:42:43,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:48,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:50,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:42:50,610 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:42:50,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:42:53,535 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:42:54,536 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:42:56,371 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:42:56,373 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:42:56,373 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:42:56,373 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:42:56,498 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:42:57,499 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:43:00,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:01,500 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:43:05,508 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:05,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:43:05,610 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:43:05,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:43:10,647 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:15,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:16,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:43:16,546 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:43:16,546 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:43:16,546 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:43:17,506 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:43:17,507 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:43:20,684 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:43:20,685 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:43:20,687 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:43:20,803 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:21,508 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:43:23,536 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:43:26,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:31,189 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:43:31,190 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:43:31,190 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:43:31,211 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:43:31,512 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:43:32,244 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:33,513 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:43:35,513 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:43:35,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:43:35,673 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:43:35,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:43:37,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:42,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:47,717 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:50,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:43:50,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:43:50,673 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:43:51,407 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:43:51,408 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:43:51,408 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:43:51,409 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:43:51,519 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:43:53,442 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:43:53,520 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:43:53,537 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:43:55,521 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:43:58,605 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:03,606 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:05,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:44:05,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:44:05,673 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:44:08,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:11,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:44:11,855 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:44:11,856 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:44:11,856 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:44:12,527 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:44:13,528 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:44:13,888 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:15,529 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:44:17,530 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:44:18,993 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:20,674 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:44:20,675 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:44:20,675 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:44:23,538 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:44:24,539 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:26,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:44:26,955 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:44:26,955 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:44:26,955 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:44:27,533 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:44:29,534 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:44:29,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:31,535 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:44:35,115 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:35,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:44:35,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:44:35,716 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:44:40,751 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:44,505 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:44:44,506 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:44:44,506 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:44:44,507 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:44:44,539 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:44:45,540 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:44:46,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:49,541 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:44:50,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:44:50,686 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:44:50,686 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:44:51,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:44:53,539 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:44:57,540 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:01,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:45:01,717 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:45:01,717 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:45:01,718 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:45:02,547 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:45:02,750 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:03,547 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:45:05,548 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:45:05,688 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:45:05,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:45:05,689 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:45:08,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:13,735 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:18,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:20,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:45:20,686 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:45:20,686 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:45:21,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:45:21,081 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:45:21,082 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:45:21,082 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:45:21,554 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:45:23,540 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:45:23,555 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:45:24,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:25,556 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:45:29,185 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:34,186 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:35,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:45:35,686 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:45:35,686 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:45:37,896 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:45:37,897 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:45:37,898 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:45:37,898 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:45:38,561 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:45:39,561 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:45:39,932 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:41,562 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:45:43,562 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:45:45,055 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:50,056 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:50,683 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:45:50,687 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:45:50,687 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:45:52,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:45:52,815 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:45:52,815 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:45:52,816 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:45:53,541 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:45:53,566 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:45:53,567 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:45:55,542 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:45:57,568 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:46:00,921 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:05,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:46:05,686 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:46:05,687 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:46:06,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:11,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:16,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:18,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:46:18,890 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:46:18,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:46:18,890 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:46:19,577 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:46:19,577 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:46:21,482 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:46:21,482 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:46:21,517 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:46:21,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:23,542 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:46:23,578 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:46:27,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:32,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:36,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:46:36,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:46:36,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:46:37,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:42,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:45,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:46:45,939 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:46:45,940 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:46:45,940 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:46:46,587 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:46:47,587 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:46:47,975 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:49,588 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:46:51,486 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:46:51,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:46:51,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:46:53,543 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:46:53,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:46:58,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:02,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:47:02,843 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:47:02,843 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:47:02,843 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:47:03,593 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:47:03,593 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:47:03,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:06,486 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:47:06,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:47:06,500 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:47:07,594 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:47:09,580 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:14,580 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:19,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:21,486 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:47:21,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:47:21,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:47:23,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:47:23,360 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:47:23,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:47:23,360 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:47:23,544 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:47:23,600 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:47:25,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:25,601 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:47:27,602 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:47:30,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:36,486 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:47:36,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:47:36,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:47:36,547 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:41,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:43,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:47:43,806 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:47:43,806 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:47:43,807 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:47:44,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:47:45,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:47:46,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:47,609 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:47:51,490 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:47:51,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:47:51,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:47:52,579 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:47:53,545 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:47:58,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:01,121 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:48:01,122 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:48:01,123 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:48:01,123 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:48:01,615 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:48:01,615 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:48:04,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:05,617 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:48:06,489 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:48:06,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:48:06,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:48:09,519 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:14,520 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:16,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:48:16,766 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:48:16,766 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:48:16,767 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:48:17,621 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:48:19,776 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:21,489 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:48:21,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:48:21,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:48:21,623 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:48:23,546 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:48:25,547 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:30,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:33,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:48:33,829 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:48:33,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:48:33,830 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:48:34,627 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:48:35,628 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:48:35,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:36,524 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:48:36,525 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:48:36,565 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:48:37,628 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:48:40,933 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:45,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:50,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:51,495 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:48:51,501 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:48:51,501 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:48:53,547 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:48:53,579 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:48:53,580 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:48:53,580 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:48:53,581 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:48:53,635 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:48:55,635 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:48:56,566 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:48:57,636 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:49:01,683 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:06,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:49:06,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:49:06,502 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:49:07,630 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:12,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:14,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:49:14,706 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:49:14,706 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:49:14,706 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:49:15,643 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:49:15,643 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:49:17,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:19,644 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:49:21,495 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:49:21,500 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:49:21,501 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:49:23,548 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:49:23,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:28,550 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:33,551 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:36,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:49:36,278 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:49:36,278 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:49:36,279 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:49:36,495 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:49:36,501 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:49:36,501 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:49:36,651 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:49:37,651 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:49:38,608 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:39,652 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:49:41,653 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:49:44,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:49,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:51,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:49:51,501 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:49:51,501 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:49:53,549 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:49:54,550 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:55,438 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:49:55,439 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:49:55,440 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:49:55,440 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:49:55,659 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:49:57,660 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:49:59,551 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:49:59,661 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:50:04,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:06,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:50:06,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:50:06,503 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:50:10,547 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:15,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:15,744 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:50:15,745 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:50:15,746 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:50:15,746 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:50:16,667 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:50:17,667 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:50:19,668 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:50:20,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:21,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:50:21,501 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:50:21,502 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:50:23,550 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:50:26,551 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:31,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:36,553 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:36,565 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:50:36,565 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:50:36,566 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:50:38,105 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:50:38,106 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:50:38,106 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:50:38,107 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:50:38,674 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:50:39,674 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:50:41,675 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:50:42,267 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:47,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:51,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:50:51,535 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:50:51,575 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:50:52,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:50:53,551 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:50:57,299 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:50:57,300 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:50:57,300 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:50:57,301 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:50:57,681 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:50:57,682 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:50:58,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:01,683 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:51:03,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:06,535 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:51:06,535 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:51:06,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:51:08,641 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:11,163 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:51:11,164 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:51:11,164 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:51:11,165 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:51:11,686 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:51:11,687 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:51:14,167 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:15,688 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:51:19,295 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:21,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:51:21,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:51:21,550 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:51:23,552 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:51:24,553 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:29,554 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:32,676 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:51:32,677 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:51:32,678 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:51:32,678 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:51:32,694 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:51:33,694 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:51:34,717 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:36,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:51:36,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:51:36,545 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:51:37,696 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:51:40,589 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:45,591 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:47,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:51:47,498 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:51:47,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:51:47,499 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:51:47,699 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:51:47,700 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:51:50,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:51:51,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:51:51,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:51:51,545 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:51:51,701 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:51:53,553 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:51:56,554 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:01,555 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:03,136 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:52:03,137 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:52:03,137 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:52:03,137 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:52:03,705 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:52:03,706 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:52:06,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:52:06,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:52:06,549 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:52:06,676 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:07,707 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:52:11,677 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:16,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:17,311 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:52:17,313 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:52:17,313 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:52:17,313 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:52:17,712 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:52:17,712 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:52:21,547 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:52:21,548 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:52:21,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:52:22,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:22,714 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:52:23,554 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:52:28,555 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:33,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:52:33,452 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:52:33,453 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:52:33,453 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:52:33,718 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:52:34,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:34,718 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:52:36,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:52:36,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:52:36,549 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:52:38,720 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:52:39,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:44,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:49,310 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:52:49,311 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:52:49,311 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:52:49,312 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:52:49,724 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:52:50,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:52:50,724 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:52:51,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:52:51,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:52:51,918 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:52:53,555 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:52:54,726 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:52:55,556 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:00,556 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:05,557 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:06,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:53:06,915 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:53:06,915 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:53:08,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:53:08,955 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:53:08,955 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:53:08,955 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:53:09,731 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:53:10,731 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:53:10,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:14,732 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:53:16,394 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:21,394 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:21,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:53:21,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:53:21,919 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:53:23,556 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:53:26,557 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:26,842 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:53:26,843 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:53:26,843 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:53:26,844 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:53:27,738 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:53:28,738 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:53:31,977 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:32,739 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:53:36,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:53:36,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:53:36,918 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:53:37,020 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:40,034 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:53:40,035 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:53:40,036 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:53:40,137 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:53:40,742 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:53:42,138 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:42,743 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:53:44,744 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:53:47,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:51,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:53:51,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:53:51,917 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:53:52,995 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:53:53,557 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:53:58,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:00,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:54:00,289 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:54:00,289 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:54:00,289 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:54:00,750 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:54:02,751 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:54:03,568 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:04,752 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:54:06,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:54:06,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:54:06,919 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:54:09,047 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:14,048 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:19,049 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:20,637 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:54:20,638 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:54:20,639 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:54:20,639 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:54:20,758 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:54:22,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:54:22,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:54:22,385 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:54:22,759 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:54:23,558 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:54:24,760 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:54:24,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:26,761 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:54:29,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:34,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:54:34,614 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:54:34,615 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:54:34,615 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:54:34,764 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:54:35,649 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:36,764 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:54:37,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:54:37,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:54:37,384 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:54:38,765 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:54:40,755 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:45,756 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:50,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:51,285 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:54:51,286 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:54:51,286 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:54:51,287 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:54:51,770 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:54:52,770 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:54:52,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:54:52,999 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:54:53,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:54:53,559 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:54:56,433 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:54:56,772 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:55:01,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:06,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:07,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:55:07,365 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:55:07,365 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:55:07,365 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:55:07,776 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:55:07,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:55:07,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:55:08,000 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:55:08,776 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:55:11,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:12,778 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:55:16,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:21,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:22,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:55:22,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:55:23,000 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:55:23,559 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:55:26,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:27,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:55:27,391 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:55:27,391 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:55:27,391 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:55:27,783 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:55:28,784 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:55:31,563 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:32,786 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:55:36,564 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:37,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:55:37,999 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:55:38,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:55:42,063 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:43,373 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:55:43,374 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:55:43,375 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:55:43,375 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:55:43,790 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:55:44,790 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:55:47,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:48,792 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:55:52,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:55:52,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:55:52,998 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:55:53,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:55:53,561 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:55:57,562 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:02,563 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:04,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:56:04,809 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:56:04,809 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:56:04,809 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:56:04,816 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:56:06,817 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:56:07,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:07,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:56:08,000 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:56:08,001 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:56:08,818 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:56:10,819 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:56:13,076 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:18,077 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:22,026 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:56:22,028 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:56:22,028 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:56:22,029 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:56:22,825 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:56:23,562 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:56:23,562 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:23,770 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:56:23,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:56:23,771 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:56:24,826 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:56:26,826 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:56:29,227 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:34,227 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:38,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:56:38,770 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:56:38,770 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:56:39,178 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:56:39,179 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:56:39,179 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:56:39,180 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:56:39,832 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:56:40,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:40,832 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:56:44,834 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:56:45,336 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:50,337 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:53,562 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:56:53,792 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:56:53,792 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:56:53,793 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:56:55,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:56:58,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:56:58,497 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:56:58,497 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:56:58,498 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:56:58,840 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:57:00,841 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:57:01,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:02,842 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:57:06,974 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:08,775 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:57:08,776 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:57:08,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:57:12,818 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:17,819 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:22,135 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:57:22,136 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:57:22,136 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:57:22,137 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:57:22,850 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:57:23,175 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:23,563 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:57:23,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:57:23,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:57:23,880 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:57:24,851 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:57:26,852 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:57:28,337 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:33,338 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:38,339 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:38,788 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:57:38,789 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:57:38,789 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:57:38,790 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:57:38,857 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:57:38,877 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:57:38,878 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:57:38,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:57:40,858 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:57:44,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:44,860 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:57:49,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:53,564 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:57:53,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:57:53,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:57:53,919 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:57:54,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:57:54,360 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:57:54,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:57:54,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:57:54,361 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:57:54,864 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:57:58,866 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:57:59,536 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:04,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:08,883 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:58:08,883 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:58:08,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:58:09,573 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:58:09,574 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:58:09,575 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:58:09,575 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:09,575 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:58:09,870 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:58:10,870 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:58:14,737 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:14,872 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:58:19,738 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:23,566 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:58:23,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:58:23,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:58:23,920 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:58:25,041 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:29,155 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:58:29,156 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:58:29,156 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:58:29,157 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:58:29,878 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:58:30,196 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:30,878 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:58:34,880 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:58:35,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:38,888 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:58:38,889 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:58:38,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:58:41,003 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:46,004 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:50,440 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:58:50,442 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:58:50,442 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:58:50,442 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:58:50,886 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:58:51,481 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:58:52,887 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:58:53,566 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:58:53,888 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:58:53,889 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:58:53,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:58:54,888 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:58:57,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:02,010 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:05,855 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:59:05,856 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:59:05,856 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:59:05,857 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:59:05,892 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:59:06,893 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:59:07,896 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:08,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:59:08,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:59:08,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:59:10,894 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:59:13,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:18,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:23,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:23,568 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:59:23,888 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:59:23,889 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:59:23,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:59:24,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:59:24,872 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:59:24,873 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:59:24,873 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:59:24,900 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:59:26,901 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:59:28,050 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:28,902 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:59:33,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:38,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:38,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:59:38,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:59:38,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:59:44,005 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:48,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 05:59:48,009 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 05:59:48,010 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 05:59:48,010 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 05:59:48,909 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 05:59:48,910 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:59:49,049 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:52,911 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 05:59:53,569 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 05:59:53,890 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 05:59:53,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 05:59:53,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 05:59:54,994 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 05:59:59,995 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:01,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:00:01,798 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:00:01,798 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:00:02,827 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:00:02,914 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:00:04,915 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:00:05,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:06,916 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:00:08,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:00:08,889 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:00:08,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:00:10,927 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:15,928 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:20,929 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:23,570 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:00:23,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:00:23,889 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:00:23,930 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:00:25,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:00:25,044 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:00:25,045 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:00:25,045 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:00:25,923 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:00:26,084 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:26,923 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:00:28,924 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:00:31,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:36,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:38,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:00:38,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:00:38,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:00:41,976 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:42,313 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:00:42,314 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:00:42,315 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:00:42,315 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:00:42,930 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:00:44,931 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:00:46,932 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:00:47,461 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:52,462 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:00:53,571 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:00:53,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:00:53,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:00:53,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:00:57,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:02,503 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:01:02,504 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:01:02,504 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:01:02,505 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:01:02,938 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:01:03,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:04,939 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:01:06,940 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:01:08,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:08,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:01:08,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:01:08,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:01:13,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:18,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:20,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:01:20,766 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:01:20,766 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:01:20,767 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:01:20,945 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:01:22,945 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:01:23,571 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:01:23,890 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:01:23,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:01:23,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:01:23,993 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:24,946 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:01:28,994 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:33,994 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:38,890 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:01:38,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:01:38,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:01:38,995 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:42,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:01:42,559 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:01:42,559 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:01:42,559 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:01:42,953 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:01:44,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:44,954 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:01:46,954 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:01:49,687 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:53,572 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:01:53,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:01:53,893 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:01:53,927 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:01:54,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:01:59,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:01,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:02:01,889 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:02:01,889 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:02:01,890 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:02:01,960 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:02:02,960 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:02:04,928 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:06,962 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:02:08,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:02:08,891 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:02:08,923 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:02:09,982 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:14,983 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:19,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:23,573 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:02:23,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:02:23,891 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:02:23,932 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:02:24,516 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:02:24,518 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:02:24,518 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:02:24,518 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:02:24,968 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:02:25,557 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:26,969 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:02:28,970 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:02:30,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:35,737 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:38,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:02:38,893 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:02:38,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:02:40,933 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:43,723 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:02:43,724 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:02:43,724 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:02:43,725 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:02:43,976 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:02:44,976 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:02:46,727 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:48,978 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:02:52,138 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:53,574 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:02:53,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:02:53,937 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:02:53,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:02:57,424 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:02:57,425 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:02:57,426 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:02:57,426 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:02:57,426 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:02:57,981 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:03:00,982 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:03:02,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:02,983 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:03:07,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:08,896 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:03:08,897 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:03:08,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:03:13,003 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:18,004 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:20,464 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:03:20,466 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:03:20,466 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:03:20,466 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:03:20,989 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:03:20,990 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:03:23,504 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:23,575 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:03:23,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:03:23,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:03:23,941 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:03:24,991 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:03:29,075 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:34,076 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:38,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:03:38,902 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:03:38,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:03:39,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:44,049 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:03:44,050 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:03:44,050 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:03:44,051 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:03:44,999 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:03:44,999 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:03:45,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:49,001 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:03:50,161 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:03:53,576 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:03:53,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:03:53,926 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:03:53,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:03:56,028 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:01,029 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:03,991 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:04:03,993 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:04:03,993 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:04:03,993 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:04:04,007 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:04:05,007 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:04:06,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:08,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:04:08,907 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:04:08,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:04:09,009 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:04:12,026 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:17,026 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:22,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:23,577 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:04:23,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:04:23,908 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:04:23,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:04:25,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:04:25,436 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:04:25,436 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:04:25,437 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:04:26,015 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:04:27,016 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:04:27,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:31,018 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:04:32,568 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:37,569 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:38,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:04:38,909 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:04:38,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:04:42,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:04:42,609 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:04:42,609 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:04:42,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:42,610 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:04:43,022 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:04:45,023 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:04:47,023 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:04:47,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:52,796 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:53,577 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:04:53,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:04:53,908 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:04:53,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:04:58,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:04:58,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:04:58,359 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:04:58,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:04:58,360 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:04:59,028 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:05:01,029 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:05:03,030 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:05:03,562 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:08,563 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:08,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:05:08,910 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:05:08,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:05:14,035 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:19,036 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:20,888 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:05:20,889 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:05:20,889 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:05:20,890 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:05:21,038 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:05:23,038 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:05:23,579 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:05:23,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:05:23,908 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:05:23,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:05:24,076 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:25,039 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:05:29,077 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:34,078 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:37,255 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:05:37,257 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:05:37,257 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:05:37,257 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:05:38,044 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:05:38,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:05:38,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:05:38,985 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:05:39,044 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:05:40,041 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:43,046 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:05:45,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:50,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:53,580 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:05:53,981 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:05:53,982 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:05:53,985 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:05:54,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:05:54,392 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:05:54,393 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:05:54,393 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:05:55,050 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:05:56,432 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:05:57,051 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:05:59,052 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:06:01,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:06,527 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:08,985 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:06:08,985 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:06:09,026 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:06:11,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:06:11,092 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:06:11,093 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:06:11,093 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:06:12,057 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:06:12,132 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:13,057 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:06:17,059 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:06:17,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:22,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:23,581 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:06:23,989 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:06:23,990 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:06:23,990 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:06:25,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:06:25,550 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:06:25,551 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:06:25,551 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:06:26,062 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:06:27,062 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:06:28,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:31,064 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:06:33,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:38,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:38,988 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:06:38,989 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:06:38,989 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:06:44,401 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:46,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:06:46,810 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:06:46,810 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:06:46,810 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:06:47,069 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:06:49,070 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:06:49,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:06:51,071 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:06:53,581 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:06:53,988 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:06:53,989 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:06:53,992 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:06:55,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:00,015 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:02,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:07:02,528 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:07:02,528 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:07:02,806 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:07:03,075 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:07:05,076 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:07:05,839 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:07,076 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:07:09,003 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:07:09,004 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:07:09,005 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:07:11,118 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:16,119 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:17,851 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:07:17,853 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:07:17,853 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:07:17,853 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:07:18,081 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:07:19,081 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:07:21,282 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:23,083 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:07:23,582 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:07:23,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:07:23,999 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:07:24,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:07:27,057 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:32,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:37,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:38,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:07:38,999 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:07:39,039 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:07:42,128 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:42,818 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:07:42,819 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:07:42,820 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:07:42,820 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:07:43,091 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:07:45,091 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:07:47,092 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:07:48,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:53,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:53,583 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:07:53,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:07:54,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:07:54,001 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:07:58,129 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:07:59,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:07:59,170 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:07:59,170 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:07:59,171 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:08:00,097 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:08:03,098 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:08:03,403 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:08,404 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:08,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:08:08,999 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:08:09,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:08:14,101 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:19,101 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:21,972 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:08:21,973 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:08:21,973 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:08:21,974 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:08:22,106 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:08:23,106 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:08:23,585 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:08:24,571 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:08:24,571 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:08:24,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:08:24,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:27,107 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:08:30,140 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:35,141 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:38,488 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:08:38,490 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:08:38,490 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:08:38,490 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:08:39,112 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:08:39,112 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:08:40,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:08:40,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:08:40,227 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:40,227 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:08:43,113 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:08:45,676 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:50,676 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:53,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:08:53,513 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:08:53,513 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:08:53,514 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:08:53,586 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:08:54,117 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:08:55,118 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:08:55,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:08:55,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:08:55,231 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:08:56,251 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:08:57,118 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:08:59,119 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:09:01,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:06,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:10,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:09:10,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:09:10,231 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:09:12,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:14,755 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:09:14,756 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:09:14,756 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:09:14,757 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:09:15,125 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:09:17,126 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:09:17,796 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:19,127 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:09:22,975 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:23,587 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:09:25,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:09:25,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:09:25,232 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:09:28,339 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:33,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:35,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:09:35,646 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:09:35,646 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:09:35,646 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:09:36,134 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:09:37,135 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:09:38,668 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:40,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:09:40,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:09:40,230 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:09:41,138 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:09:44,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:49,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:53,588 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:09:54,589 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:09:55,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:09:55,230 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:09:55,271 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:09:59,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:09:59,547 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:09:59,548 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:09:59,548 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:10:00,145 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:10:00,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:01,146 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:10:03,146 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:10:05,147 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:10:05,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:10,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:10:10,230 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:10:10,232 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:10:11,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:16,357 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:21,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:23,588 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:10:25,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:10:25,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:10:25,236 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:10:26,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:10:26,270 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:10:26,270 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:10:26,270 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:10:27,156 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:10:27,156 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:10:27,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:31,157 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:10:32,411 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:37,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:40,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:10:40,235 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:10:40,236 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:10:43,284 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:45,373 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:10:45,375 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:10:45,375 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:10:45,375 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:10:46,168 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:10:47,168 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:10:48,407 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:49,169 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:10:53,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:10:53,589 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:10:55,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:10:55,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:10:55,235 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:10:59,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:03,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:11:03,047 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:11:03,047 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:11:03,048 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:11:03,174 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:11:05,087 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:06,175 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:11:08,177 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:11:10,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:10,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:11:10,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:11:10,237 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:11:15,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:20,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:23,591 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:11:25,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:11:25,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:11:25,235 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:11:25,338 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:27,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:11:27,513 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:11:27,514 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:11:27,514 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:11:28,184 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:11:30,184 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:11:30,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:32,185 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:11:35,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:40,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:11:40,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:11:40,235 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:11:41,261 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:42,833 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:11:42,835 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:11:42,835 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:11:42,835 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:11:43,189 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:11:46,190 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:11:47,066 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:48,191 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:11:52,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:53,592 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:11:55,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:11:55,239 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:11:55,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:11:57,292 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:11:59,142 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:11:59,143 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:11:59,143 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:11:59,153 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:11:59,195 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:12:02,196 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:12:02,371 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:04,197 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:12:07,372 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:10,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:12:10,235 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:12:10,236 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:12:13,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:17,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:12:17,611 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:12:17,612 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:12:17,612 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:12:18,202 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:12:18,642 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:20,203 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:12:22,203 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:12:23,593 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:12:24,593 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:25,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:12:25,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:12:25,238 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:12:30,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:35,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:40,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:12:40,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:12:40,238 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:12:41,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:41,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:12:41,523 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:12:41,524 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:12:41,524 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:12:42,212 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:12:44,213 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:12:46,214 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:12:46,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:51,668 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:12:53,594 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:12:55,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:12:55,235 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:12:55,236 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:12:57,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:01,752 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:13:01,753 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:13:01,754 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:13:01,754 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:13:02,220 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:13:02,793 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:04,221 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:13:06,222 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:13:07,876 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:10,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:13:10,235 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:13:10,236 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:13:13,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:18,320 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:19,062 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:13:19,068 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:13:19,069 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:13:19,069 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:13:19,226 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:13:22,227 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:13:23,595 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:13:23,595 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:24,228 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:13:25,280 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:13:25,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:13:25,281 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:13:29,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:34,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:37,337 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:13:37,339 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:13:37,339 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:13:37,339 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:13:38,234 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:13:39,382 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:40,234 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:13:40,257 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:13:40,257 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:13:40,260 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:13:42,235 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:13:44,520 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:49,521 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:53,596 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:13:54,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:13:55,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:13:55,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:13:55,242 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:14:00,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:02,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:14:02,087 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:14:02,088 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:14:02,088 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:14:02,243 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:14:04,244 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:14:05,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:08,246 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:14:10,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:14:10,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:14:10,244 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:14:11,294 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:16,295 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:21,296 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:21,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:14:21,831 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:14:21,831 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:14:21,831 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:14:22,252 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:14:23,597 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:14:24,253 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:14:25,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:14:25,247 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:14:25,267 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:14:26,253 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:14:26,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:31,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:36,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:14:36,291 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:14:36,291 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:14:36,292 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:14:37,258 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:14:37,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:38,258 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:14:40,249 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:14:40,250 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:14:40,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:14:42,260 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:14:42,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:47,370 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:52,370 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:53,597 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:14:55,249 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:14:55,249 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:14:55,251 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:14:58,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:14:58,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:14:58,664 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:14:58,664 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:14:58,664 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:14:59,266 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:15:00,266 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:15:03,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:04,268 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:15:08,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:10,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:15:10,247 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:15:10,249 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:15:14,375 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:19,376 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:21,120 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:15:21,121 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:15:21,122 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:15:21,122 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:15:21,274 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:15:23,598 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:15:24,275 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:15:25,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:15:25,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:15:25,252 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:15:25,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:26,276 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:15:30,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:35,079 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:15:35,080 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:15:35,080 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:15:35,081 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:15:35,280 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:15:36,114 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:36,280 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:15:40,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:15:40,248 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:15:40,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:15:40,281 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:15:41,312 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:46,313 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:51,314 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:53,599 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:15:55,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:15:55,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:15:55,269 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:15:55,732 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:15:55,733 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:15:55,733 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:15:55,733 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:15:56,287 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:15:56,773 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:15:58,288 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:16:00,289 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:16:01,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:06,883 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:10,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:16:10,254 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:16:10,259 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:16:12,373 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:16,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:16:16,546 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:16:16,546 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:16:16,547 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:16:17,295 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:16:17,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:18,296 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:16:22,297 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:16:22,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:23,599 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:16:25,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:16:25,253 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:16:25,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:16:28,312 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:33,312 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:38,313 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:40,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:16:40,253 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:16:40,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:16:42,524 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:16:42,525 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:16:42,525 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:16:42,525 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:16:43,305 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:16:43,557 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:44,305 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:16:48,307 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:16:48,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:53,600 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:16:54,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:16:55,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:16:55,254 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:16:55,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:17:00,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:00,467 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:17:00,468 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:17:00,468 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:17:00,469 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:17:01,311 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:17:02,312 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:17:05,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:06,314 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:17:10,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:17:10,254 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:17:10,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:17:11,338 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:16,339 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:18,298 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:17:18,299 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:17:18,300 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:17:18,300 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:17:18,318 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:17:20,319 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:17:21,346 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:23,601 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:17:24,320 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:17:25,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:17:25,254 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:17:25,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:17:26,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:31,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:35,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:17:35,981 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:17:35,981 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:17:35,982 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:17:36,324 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:17:37,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:38,325 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:17:40,272 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:17:40,272 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:17:40,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:17:40,326 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:17:42,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:47,306 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:52,307 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:53,480 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:17:53,481 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:17:53,481 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:17:53,482 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:17:53,602 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:17:54,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:17:56,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:17:56,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:17:56,091 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:17:56,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:17:57,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:17:58,332 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:18:02,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:07,647 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:11,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:18:11,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:18:11,090 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:18:13,157 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:16,129 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:18:16,130 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:18:16,131 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:18:16,131 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:18:16,338 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:18:18,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:18,339 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:18:20,340 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:18:23,329 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:23,608 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:18:26,093 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:18:26,093 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:18:26,133 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:18:29,224 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:34,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:18:34,086 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:18:34,086 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:18:34,086 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:18:34,344 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:18:35,125 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:36,345 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:18:38,346 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:18:40,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:41,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:18:41,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:18:41,090 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:18:46,192 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:51,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:51,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:18:51,668 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:18:51,668 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:18:51,668 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:18:52,352 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:18:53,605 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:18:54,352 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:18:56,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:18:56,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:18:56,090 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:18:56,208 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:18:56,353 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:19:01,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:06,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:08,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:19:08,171 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:19:08,172 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:19:08,172 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:19:08,358 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:19:10,359 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:19:11,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:19:11,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:19:11,090 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:19:11,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:12,360 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:19:16,382 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:21,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:23,606 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:19:26,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:19:26,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:19:26,091 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:19:27,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:29,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:19:29,248 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:19:29,248 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:19:29,250 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:19:29,366 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:19:30,367 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:19:32,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:34,369 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:19:37,409 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:41,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:19:41,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:19:41,092 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:19:43,127 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:45,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:19:45,070 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:19:45,071 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:19:45,071 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:19:45,373 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:19:46,373 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:19:48,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:50,375 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:19:53,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:19:53,606 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:19:56,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:19:56,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:19:56,090 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:19:59,191 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:04,192 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:07,481 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:20:07,482 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:20:07,483 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:20:07,485 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:20:08,381 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:20:08,381 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:20:09,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:11,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:20:11,093 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:20:11,094 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:20:12,383 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:20:15,211 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:20,212 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:23,055 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:20:23,056 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:20:23,056 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:20:23,056 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:20:23,387 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:20:23,608 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:20:24,387 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:20:25,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:26,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:20:26,093 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:20:26,093 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:20:28,389 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:20:31,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:36,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:41,098 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:20:41,099 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:20:41,099 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:20:42,147 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:43,482 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:20:43,483 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:20:43,483 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:20:43,486 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:20:44,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:20:44,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:20:47,640 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:48,397 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:20:52,641 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:20:53,609 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:20:56,095 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:20:56,096 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:20:56,097 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:20:58,166 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:03,167 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:03,981 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:21:03,982 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:21:03,983 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:21:03,983 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:21:04,403 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:21:06,404 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:21:08,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:08,404 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:21:11,096 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:21:11,097 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:21:11,097 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:21:13,232 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:18,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:23,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:23,610 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:21:25,056 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:21:25,057 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:21:25,057 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:21:25,058 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:21:25,411 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:21:26,411 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:21:26,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:21:26,781 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:21:26,781 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:21:29,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:30,413 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:21:34,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:39,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:41,781 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:21:41,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:21:41,783 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:21:44,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:46,316 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:21:46,317 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:21:46,317 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:21:46,317 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:21:46,419 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:21:48,420 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:21:50,421 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:21:50,470 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:53,610 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:21:55,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:21:56,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:21:56,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:21:56,780 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:22:00,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:05,696 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:22:05,697 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:22:05,697 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:22:05,698 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:22:06,427 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:22:06,737 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:08,428 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:22:10,429 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:22:11,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:22:11,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:22:11,780 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:22:11,856 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:16,857 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:21,857 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:23,612 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:22:24,064 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:22:24,065 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:22:24,065 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:22:24,066 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:22:24,434 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:22:26,435 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:22:26,794 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:22:26,794 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:22:26,795 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:22:27,094 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:28,436 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:22:32,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:37,222 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:41,784 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:22:41,785 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:22:41,789 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:22:42,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:43,632 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:22:43,633 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:22:43,634 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:22:43,634 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:22:44,442 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:22:44,443 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:22:48,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:48,444 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:22:53,087 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:22:53,612 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:22:56,785 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:22:56,785 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:22:56,790 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:22:58,893 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:03,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:03,922 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:23:03,923 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:23:03,923 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:23:03,924 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:23:04,451 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:23:06,452 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:23:08,453 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:23:09,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:11,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:23:11,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:23:11,921 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:23:14,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:19,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:23,613 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:23:25,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:26,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:23:26,903 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:23:26,944 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:23:27,136 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:23:27,138 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:23:27,138 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:23:27,138 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:23:27,460 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:23:28,460 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:23:31,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:32,462 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:23:36,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:41,301 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:41,325 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:23:41,326 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:23:41,326 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:23:41,326 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:23:41,465 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:23:41,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:23:41,903 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:23:41,944 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:23:42,466 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:23:46,467 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:23:46,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:51,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:53,614 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:23:56,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:23:56,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:23:56,905 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:23:56,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:23:58,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:23:58,600 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:23:58,600 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:23:58,600 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:23:59,472 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:24:00,473 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:02,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:02,474 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:04,474 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:07,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:11,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:24:11,905 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:24:11,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:24:12,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:17,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:22,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:24:22,547 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:24:22,547 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:24:22,548 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:24:23,481 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:24:23,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:23,615 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:24:24,481 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:26,482 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:26,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:24:26,904 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:24:26,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:24:28,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:33,987 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:38,988 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:39,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:24:39,242 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:24:39,242 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:24:39,242 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:24:39,486 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:24:40,487 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:41,991 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:24:41,992 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:24:41,992 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:24:44,399 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:44,488 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:49,400 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:53,616 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:24:53,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:24:53,646 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:24:53,646 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:24:53,647 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:24:54,492 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:24:54,492 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:54,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:24:56,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:24:56,909 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:24:56,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:24:58,494 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:24:59,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:04,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:09,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:11,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:25:11,044 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:25:11,044 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:25:11,045 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:25:11,499 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:25:11,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:25:11,909 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:25:11,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:25:12,499 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:25:15,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:16,501 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:25:20,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:23,617 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:25:25,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:26,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:25:26,910 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:25:26,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:25:31,041 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:35,844 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:25:35,845 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:25:35,845 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:25:35,846 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:25:36,509 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:25:36,509 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:25:36,885 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:40,511 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:25:41,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:41,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:25:41,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:25:41,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:25:47,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:52,028 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:25:52,591 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:25:52,592 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:25:52,592 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:25:52,593 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:25:53,516 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:25:53,617 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:25:54,516 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:25:56,517 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:25:56,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:25:56,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:25:56,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:25:58,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:03,015 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:08,016 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:11,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:26:11,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:26:11,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:26:12,711 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:26:12,712 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:26:12,712 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:26:12,713 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:26:13,523 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:26:13,752 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:14,524 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:26:16,525 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:26:18,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:23,618 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:26:24,619 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:26,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:26:26,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:26:26,951 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:26:29,971 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:33,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:26:33,681 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:26:33,681 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:26:33,681 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:26:34,532 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:26:34,532 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:26:35,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:38,534 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:26:40,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:41,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:26:41,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:26:41,951 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:26:46,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:49,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:26:49,459 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:26:49,459 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:26:49,460 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:26:49,541 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:26:50,542 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:26:51,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:53,619 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:26:54,543 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:26:56,620 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:26:56,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:26:56,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:26:56,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:27:01,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:03,759 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:27:03,760 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:27:03,760 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:27:03,761 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:27:04,548 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:27:04,549 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:27:06,951 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:08,550 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:27:11,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:27:11,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:27:11,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:27:12,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:17,028 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:22,029 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:23,620 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:27:26,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:27:26,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:27:26,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:27:27,039 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:28,427 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:27:28,429 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:27:28,429 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:27:28,429 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:27:28,557 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:27:30,558 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:27:32,559 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:27:32,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:37,630 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:41,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:27:41,912 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:27:41,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:27:42,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:45,007 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:27:45,009 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:27:45,009 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:27:45,009 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:27:45,563 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:27:46,563 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:27:48,049 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:48,564 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:27:53,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:27:53,621 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:27:56,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:27:56,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:27:56,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:27:58,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:02,176 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:28:02,177 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:28:02,178 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:28:02,178 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:28:02,570 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:28:02,570 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:28:04,217 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:06,572 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:28:09,357 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:11,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:28:11,912 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:28:11,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:28:15,064 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:20,065 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:20,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:28:20,938 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:28:20,938 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:28:20,939 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:28:21,578 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:28:22,579 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:28:23,622 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:28:24,579 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:28:25,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:26,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:28:26,912 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:28:26,921 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:28:30,956 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:35,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:37,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:28:37,921 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:28:37,922 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:28:37,922 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:28:38,585 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:28:38,585 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:28:40,962 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:41,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:28:41,912 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:28:41,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:28:42,587 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:28:45,997 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:50,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:53,624 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:28:56,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:28:56,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:28:56,912 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:28:56,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:28:58,801 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:28:58,802 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:28:58,802 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:28:58,803 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:28:59,593 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:29:01,595 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:29:01,842 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:03,595 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:29:07,047 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:11,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:29:11,912 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:29:11,921 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:29:12,974 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:17,975 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:21,223 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:29:21,224 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:29:21,224 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:29:21,225 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:29:21,604 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:29:23,263 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:23,605 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:29:23,625 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:29:25,606 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:29:26,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:29:26,913 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:29:26,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:29:27,607 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:29:28,981 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:33,982 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:38,983 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:41,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:29:41,913 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:29:41,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:29:44,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:44,313 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:29:44,314 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:29:44,315 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:29:44,315 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:29:44,613 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:29:45,613 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:29:49,461 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:49,615 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:29:53,626 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:29:54,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:29:56,942 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:29:56,943 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:29:56,943 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:30:00,079 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:02,162 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:30:02,163 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:30:02,163 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:30:02,164 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:30:02,621 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:30:03,621 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:30:05,183 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:07,623 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:30:10,314 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:11,927 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:30:11,929 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:30:11,971 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:30:16,059 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:21,060 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:22,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:30:22,981 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:30:22,981 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:30:22,982 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:30:23,627 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:30:23,630 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:30:25,631 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:30:26,146 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:26,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:30:26,940 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:30:26,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:30:27,632 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:30:32,044 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:37,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:38,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:30:38,665 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:30:38,666 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:30:38,666 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:30:39,637 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:30:41,638 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:30:41,932 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:30:41,932 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:30:41,973 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:30:42,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:43,639 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:30:47,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:53,000 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:53,628 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:30:54,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:30:54,214 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:30:54,214 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:30:54,214 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:30:54,643 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:30:55,643 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:30:57,016 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:30:57,016 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:30:57,017 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:30:58,453 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:30:59,645 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:31:03,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:08,455 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:11,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:31:11,954 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:31:11,995 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:31:13,022 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:31:13,023 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:31:13,023 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:31:13,024 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:31:13,656 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:31:14,063 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:15,657 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:31:17,659 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:31:19,164 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:23,629 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:31:24,630 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:26,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:31:26,952 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:31:26,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:31:29,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:34,987 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:39,988 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:41,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:31:41,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:31:41,954 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:31:42,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:31:42,368 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:31:42,369 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:31:42,369 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:31:42,668 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:31:43,669 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:31:45,378 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:47,670 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:31:50,490 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:53,630 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:31:55,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:31:56,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:31:56,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:31:56,953 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:31:59,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:31:59,815 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:31:59,815 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:31:59,816 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:32:00,675 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:32:00,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:01,676 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:32:05,677 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:32:05,993 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:10,994 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:11,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:32:11,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:32:11,953 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:32:16,042 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:20,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:32:20,872 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:32:20,872 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:32:20,872 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:32:21,682 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:32:21,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:23,631 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:32:23,683 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:32:25,684 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:32:26,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:26,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:32:26,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:32:26,953 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:32:32,039 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:37,040 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:41,955 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:32:41,955 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:32:41,996 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:32:43,010 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:43,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:32:43,772 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:32:43,773 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:32:43,773 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:32:44,692 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:32:45,692 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:32:48,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:49,694 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:32:53,632 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:32:54,633 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:32:56,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:32:56,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:32:56,954 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:33:00,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:04,128 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:33:04,130 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:33:04,130 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:33:04,130 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:33:04,699 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:33:05,166 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:05,700 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:33:09,701 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:33:10,311 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:11,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:33:11,955 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:33:11,996 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:33:16,077 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:20,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:33:20,111 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:33:20,111 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:33:20,111 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:33:20,706 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:33:21,151 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:21,706 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:33:23,633 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:33:25,708 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:33:26,634 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:26,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:33:26,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:33:26,955 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:33:31,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:36,393 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:33:36,394 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:33:36,395 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:33:36,395 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:33:36,713 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:33:37,426 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:37,713 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:33:41,715 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:33:41,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:33:41,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:33:41,954 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:33:43,087 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:48,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:53,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:53,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:33:53,603 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:33:53,603 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:33:53,604 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:33:53,633 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:33:53,720 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:33:55,721 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:33:56,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:33:56,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:33:56,954 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:33:57,722 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:33:59,041 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:33:59,723 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:34:04,042 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:09,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:10,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:34:10,864 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:34:10,865 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:34:10,865 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:34:11,727 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:34:12,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:34:12,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:34:12,604 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:34:13,728 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:34:14,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:15,729 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:34:19,057 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:23,634 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:34:24,635 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:27,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:34:27,086 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:34:27,086 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:34:27,087 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:34:27,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:34:27,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:34:27,602 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:34:27,733 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:34:29,734 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:34:29,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:31,735 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:34:35,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:40,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:42,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:34:42,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:34:42,603 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:34:45,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:47,024 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:34:47,025 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:34:47,025 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:34:47,026 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:34:47,741 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:34:49,742 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:34:51,411 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:51,743 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:34:53,635 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:34:56,636 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:34:57,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:34:57,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:34:57,603 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:35:01,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:06,705 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:07,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:35:07,221 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:35:07,221 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:35:07,221 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:35:07,749 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:35:09,750 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:35:11,751 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:35:12,406 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:12,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:35:12,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:35:12,603 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:35:17,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:22,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:23,636 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:35:27,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:35:27,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:35:27,603 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:35:27,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:32,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:33,063 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:35:33,064 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:35:33,065 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:35:33,065 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:35:33,758 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:35:35,759 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:35:37,760 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:35:38,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:42,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:35:42,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:35:42,603 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:35:43,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:48,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:50,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:35:50,216 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:35:50,216 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:35:50,216 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:35:50,765 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:35:51,766 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:35:53,637 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:35:54,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:35:55,767 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:35:57,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:35:57,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:35:57,603 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:35:59,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:04,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:09,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:12,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:36:12,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:36:12,604 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:36:12,649 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:36:12,701 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:36:12,701 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:36:12,702 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:36:12,774 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:36:13,774 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:36:15,685 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:17,776 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:36:20,839 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:23,638 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:36:26,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:27,604 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:36:27,608 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:36:27,608 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:36:27,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:36:27,888 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:36:27,888 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:36:27,888 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:36:28,780 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:36:29,780 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:36:31,781 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:36:32,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:33,782 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:36:37,059 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:42,060 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:42,604 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:36:42,608 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:36:42,608 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:36:44,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:36:44,658 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:36:44,658 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:36:44,658 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:36:44,786 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:36:45,787 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:36:47,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:49,789 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:36:53,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:36:53,639 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:36:57,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:36:57,619 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:36:57,619 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:36:58,708 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:03,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:04,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:37:04,070 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:37:04,071 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:37:04,071 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:37:04,794 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:37:05,794 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:37:09,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:09,796 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:37:12,606 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:37:12,608 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:37:12,608 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:37:14,753 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:19,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:20,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:37:20,222 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:37:20,223 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:37:20,223 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:37:20,800 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:37:21,801 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:37:23,639 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:37:25,640 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:25,802 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:37:27,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:37:27,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:37:27,608 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:37:30,733 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:35,735 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:40,564 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:37:40,565 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:37:40,565 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:37:40,566 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:37:40,807 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:37:41,608 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:41,808 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:37:43,225 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:37:43,225 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:37:43,226 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:37:45,810 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:37:46,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:51,850 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:53,640 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:37:57,642 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:37:58,263 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:37:58,263 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:37:58,264 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:37:58,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:37:58,721 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:37:58,721 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:37:58,721 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:37:58,815 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:37:59,815 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:38:02,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:03,817 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:38:07,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:12,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:13,228 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:38:13,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:38:13,229 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:38:18,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:21,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:38:21,386 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:38:21,386 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:38:21,386 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:38:21,824 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:38:23,418 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:23,641 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:38:23,825 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:38:25,825 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:38:28,228 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:38:28,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:38:28,229 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:38:29,323 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:34,324 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:39,325 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:40,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:38:40,895 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:38:40,896 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:38:40,896 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:38:41,831 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:38:41,832 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:38:43,557 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:38:43,558 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:38:43,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:38:44,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:47,834 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:38:50,525 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:53,642 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:38:55,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:38:58,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:38:58,217 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:38:58,217 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:38:58,217 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:38:58,555 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:38:58,556 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:38:58,556 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:38:58,840 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:38:59,840 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:39:01,280 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:03,842 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:39:06,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:11,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:13,557 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:39:13,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:39:13,558 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:39:16,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:21,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:22,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:39:22,985 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:39:22,985 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:39:22,986 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:39:23,643 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:39:23,851 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:39:23,851 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:39:27,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:27,852 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:39:28,555 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:39:28,556 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:39:28,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:39:32,598 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:37,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:41,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:39:41,028 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:39:41,029 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:39:41,029 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:39:41,857 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:39:41,858 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:39:43,068 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:43,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:39:43,668 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:39:43,668 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:39:45,859 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:39:48,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:53,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:53,647 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:39:58,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:39:58,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:39:58,666 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:39:58,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:40:00,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:40:00,306 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:40:00,306 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:40:00,306 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:40:00,865 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:40:01,866 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:40:03,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:05,867 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:40:08,723 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:13,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:40:13,667 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:40:13,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:40:13,731 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:18,732 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:23,644 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:40:24,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:28,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:40:28,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:40:28,667 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:40:29,724 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:31,724 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:40:31,726 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:40:31,726 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:40:31,728 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:40:31,879 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:40:33,880 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:40:34,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:35,881 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:40:39,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:43,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:40:43,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:40:43,667 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:40:45,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:47,483 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:40:47,484 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:40:47,484 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:40:47,485 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:40:47,887 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:40:49,888 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:40:50,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:51,889 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:40:53,645 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:40:56,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:40:58,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:40:58,669 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:40:58,671 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:41:01,778 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:03,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:41:03,917 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:41:03,918 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:41:03,920 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:41:04,894 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:41:05,894 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:41:06,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:07,895 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:41:12,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:13,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:41:13,668 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:41:13,708 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:41:17,773 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:22,774 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:23,646 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:41:27,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:41:27,707 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:41:27,708 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:41:27,708 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:41:27,902 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:41:28,747 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:29,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:41:29,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:41:29,448 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:41:29,903 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:41:31,904 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:41:33,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:38,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:43,915 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:44,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:41:44,446 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:41:44,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:41:46,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:41:46,827 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:41:46,828 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:41:46,830 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:41:46,910 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:41:47,910 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:41:49,844 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:51,912 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:41:53,647 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:41:55,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:41:59,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:41:59,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:41:59,447 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:42:01,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:06,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:06,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:42:06,909 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:42:06,910 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:42:06,910 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:42:06,918 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:42:07,918 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:42:11,920 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:42:12,072 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:14,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:42:14,449 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:42:14,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:42:17,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:22,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:42:22,333 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:42:22,333 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:42:22,336 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:42:22,924 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:42:23,372 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:23,648 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:42:23,924 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:42:25,925 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:42:28,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:29,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:42:29,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:42:29,448 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:42:33,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:38,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:40,500 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:42:40,502 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:42:40,502 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:42:40,502 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:42:40,931 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:42:41,931 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:42:43,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:44,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:42:44,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:42:44,449 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:42:45,933 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:42:49,557 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:53,649 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:42:54,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:42:59,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:42:59,447 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:42:59,488 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:43:00,495 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:02,761 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:43:02,762 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:43:02,763 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:43:02,763 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:43:02,941 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:43:03,941 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:43:05,802 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:07,943 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:43:11,203 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:14,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:43:14,447 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:43:14,450 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:43:16,554 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:21,554 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:22,152 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:43:22,153 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:43:22,153 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:43:22,153 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:43:22,949 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:43:23,649 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:43:23,949 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:43:25,950 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:43:27,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:29,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:43:29,447 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:43:29,450 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:43:32,485 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:36,729 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:43:36,730 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:43:36,730 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:43:36,730 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:43:36,955 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:43:37,770 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:37,955 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:43:41,957 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:43:42,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:44,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:43:44,447 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:43:44,450 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:43:48,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:53,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:53,650 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:43:56,776 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:43:56,781 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:43:56,781 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:43:56,781 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:43:56,963 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:43:57,964 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:43:58,817 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:43:59,525 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:43:59,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:43:59,526 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:44:01,966 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:44:03,956 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:08,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:11,400 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:44:11,402 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:44:11,402 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:44:11,402 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:44:11,970 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:44:11,970 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:44:14,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:14,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:44:14,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:44:14,496 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:44:15,971 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:44:19,616 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:23,652 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:44:24,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:26,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:44:26,238 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:44:26,238 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:44:26,239 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:44:26,975 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:44:27,976 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:44:29,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:44:29,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:44:29,458 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:44:29,976 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:44:30,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:35,575 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:40,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:44,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:44:44,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:44:44,477 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:44:46,532 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:49,137 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:44:49,139 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:44:49,139 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:44:49,139 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:44:49,985 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:44:49,985 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:44:52,178 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:53,652 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:44:53,986 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:44:57,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:44:59,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:44:59,463 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:44:59,464 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:45:03,485 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:07,084 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:45:07,085 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:45:07,085 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:45:07,086 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:45:07,991 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:45:07,992 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:45:09,125 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:11,993 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:45:14,261 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:14,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:45:14,463 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:45:14,463 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:45:19,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:23,654 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:45:24,655 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:26,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:45:26,862 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:45:26,862 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:45:26,863 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:45:26,999 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:45:27,999 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:45:29,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:45:29,585 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:45:29,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:45:29,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:32,001 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:45:34,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:39,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:44,501 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:45:44,502 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:45:44,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:45:45,615 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:45,975 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:45:45,975 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:45:45,976 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:45:45,976 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:45:46,006 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:45:48,007 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:45:50,008 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:45:51,145 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:53,655 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:45:56,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:45:59,501 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:45:59,502 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:45:59,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:46:02,608 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:06,424 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:46:06,426 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:46:06,426 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:46:06,426 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:46:07,014 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:46:08,015 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:46:08,465 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:10,016 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:46:13,641 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:14,501 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:46:14,502 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:46:14,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:46:19,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:23,656 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:46:24,657 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:26,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:46:26,653 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:46:26,654 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:46:26,654 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:46:27,021 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:46:28,022 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:46:29,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:46:29,502 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:46:29,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:46:29,696 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:30,023 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:46:34,815 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:39,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:44,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:46:44,034 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:46:44,034 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:46:44,035 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:46:44,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:46:44,502 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:46:44,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:46:45,029 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:46:45,570 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:46,029 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:46:48,031 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:46:51,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:53,656 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:46:56,657 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:46:59,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:46:59,503 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:46:59,508 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:47:02,596 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:04,461 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:47:04,463 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:47:04,463 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:47:04,463 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:47:05,038 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:47:07,039 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:47:07,616 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:09,040 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:47:12,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:14,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:47:14,503 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:47:14,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:47:18,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:23,547 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:23,657 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:47:26,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:47:26,770 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:47:26,770 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:47:26,771 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:47:27,047 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:47:28,810 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:29,048 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:47:29,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:47:29,503 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:47:29,508 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:47:31,049 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:47:33,901 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:38,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:42,475 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:47:42,476 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:47:42,476 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:47:42,477 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:47:43,064 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:47:44,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:45,065 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:47:45,102 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:47:45,102 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:47:45,102 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:47:47,065 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:47:49,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:53,658 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:47:55,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:47:58,475 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:47:58,476 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:47:58,477 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:47:58,477 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:47:59,070 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:48:00,178 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:48:00,179 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:48:00,179 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:48:01,071 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:48:01,207 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:03,072 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:48:06,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:11,649 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:14,800 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:48:14,801 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:48:14,802 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:48:14,802 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:48:15,077 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:48:15,178 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:48:15,178 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:48:15,178 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:48:17,077 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:48:17,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:21,079 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:48:23,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:23,659 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:48:28,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:30,179 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:48:30,180 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:48:30,182 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:48:34,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:34,630 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:48:34,632 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:48:34,632 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:48:34,632 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:48:35,084 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:48:37,085 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:48:39,086 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:48:39,750 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:44,750 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:45,179 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:48:45,180 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:48:45,182 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:48:49,293 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:48:49,295 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:48:49,295 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:48:49,295 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:48:50,090 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:48:50,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:48:51,091 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:48:53,660 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:48:55,093 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:48:55,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:00,179 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:49:00,180 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:49:00,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:49:01,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:06,270 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:11,271 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:14,003 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:49:14,004 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:49:14,005 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:49:14,005 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:49:14,100 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:49:15,101 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:49:15,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:49:15,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:49:15,755 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:49:16,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:19,102 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:49:22,208 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:23,661 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:49:27,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:30,753 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:49:30,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:49:30,754 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:49:31,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:49:31,829 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:49:31,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:49:31,830 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:49:32,107 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:49:32,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:33,108 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:49:37,110 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:49:37,977 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:42,978 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:45,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:49:45,754 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:49:45,794 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:49:48,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:49,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:49:49,310 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:49:49,311 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:49:49,311 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:49:50,115 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:49:51,115 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:49:53,662 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:49:54,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:49:55,117 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:49:59,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:00,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:50:00,754 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:50:00,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:50:04,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:06,803 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:50:06,805 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:50:06,805 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:50:06,805 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:50:07,121 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:50:09,121 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:50:09,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:11,122 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:50:14,987 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:16,061 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:50:16,063 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:50:16,064 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:50:20,105 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:22,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:50:22,630 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:50:22,631 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:50:22,631 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:50:23,126 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:50:23,663 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:50:25,127 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:50:25,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:27,128 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:50:30,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:31,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:50:31,008 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:50:31,012 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:50:36,039 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:39,763 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:50:39,764 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:50:39,764 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:50:39,764 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:50:40,133 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:50:41,133 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:50:41,804 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:45,135 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:50:46,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:50:46,008 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:50:46,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:50:47,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:52,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:53,664 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:50:54,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:50:54,082 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:50:54,082 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:50:54,083 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:50:54,138 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:50:55,139 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:50:57,122 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:50:59,140 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:51:01,007 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:51:01,008 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:51:01,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:51:03,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:08,068 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:10,198 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:51:10,199 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:51:10,199 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:51:10,199 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:51:11,145 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:51:11,145 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:51:13,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:15,146 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:51:16,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:51:16,008 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:51:16,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:51:19,078 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:23,665 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:51:24,665 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:29,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:30,411 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:51:30,413 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:51:30,413 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:51:30,413 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:51:31,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:51:31,009 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:51:31,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:51:31,153 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:51:33,153 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:51:35,154 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:51:35,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:40,598 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:45,598 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:46,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:51:46,009 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:51:46,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:51:49,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:51:49,346 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:51:49,346 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:51:49,346 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:51:50,159 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:51:51,160 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:51:51,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:51:53,665 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:51:55,161 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:51:56,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:01,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:52:01,009 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:52:01,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:52:02,100 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:03,593 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:52:03,594 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:52:03,595 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:52:03,595 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:52:04,164 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:52:05,165 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:52:07,792 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:09,166 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:52:12,793 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:16,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:52:16,009 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:52:16,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:52:18,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:21,171 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:52:21,172 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:52:21,172 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:52:21,173 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:52:22,171 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:52:23,171 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:52:23,212 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:23,666 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:52:25,172 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:52:28,316 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:31,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:52:31,009 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:52:31,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:52:34,111 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:39,112 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:40,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:52:40,661 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:52:40,662 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:52:40,662 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:52:41,177 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:52:43,178 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:52:44,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:45,179 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:52:46,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:52:46,010 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:52:46,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:52:50,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:52:53,667 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:52:55,668 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:00,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:01,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:53:01,009 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:53:01,050 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:53:03,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:53:03,244 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:53:03,244 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:53:03,245 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:53:04,186 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:53:05,187 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:53:06,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:07,188 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:53:11,396 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:16,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:53:16,009 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:53:16,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:53:17,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:22,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:23,021 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:53:23,022 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:53:23,022 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:53:23,023 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:53:23,193 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:53:23,668 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:53:25,194 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:53:27,195 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:27,196 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:53:31,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:53:31,010 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:53:31,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:53:33,044 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:38,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:40,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:53:40,723 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:53:40,723 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:53:40,724 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:53:41,201 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:53:43,202 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:53:43,762 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:45,203 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:53:46,011 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:53:46,011 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:53:46,015 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:53:49,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:53,669 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:53:54,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:53:58,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:53:58,316 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:53:58,317 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:53:58,317 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:53:59,216 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:53:59,216 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:00,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:01,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:54:01,073 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:54:01,074 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:54:03,217 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:05,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:10,535 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:12,992 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:54:12,993 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:54:12,993 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:54:12,994 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:54:13,221 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:54:15,222 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:16,026 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:16,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:54:16,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:54:16,033 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:54:17,223 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:21,244 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:23,670 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:54:26,671 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:30,973 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:54:30,974 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:54:30,975 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:54:30,975 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:54:31,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:54:31,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:54:31,030 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:54:31,228 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:54:32,164 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:33,229 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:35,230 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:37,231 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:37,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:42,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:46,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:54:46,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:54:46,031 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:54:47,555 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:54:47,556 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:54:47,557 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:54:47,557 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:54:48,235 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:54:48,596 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:49,235 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:53,237 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:54:53,671 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:54:53,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:54:58,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:01,028 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:55:01,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:55:01,030 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:55:04,114 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:09,115 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:09,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:55:09,337 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:55:09,337 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:55:09,338 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:55:10,244 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:55:11,244 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:55:13,245 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:55:14,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:16,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:55:16,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:55:16,031 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:55:20,143 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:23,672 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:55:25,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:26,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:55:26,720 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:55:26,720 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:55:26,721 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:55:27,251 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:55:29,252 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:55:30,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:31,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:55:31,034 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:55:31,035 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:55:31,253 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:55:36,144 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:41,145 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:45,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:55:45,798 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:55:45,798 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:55:45,799 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:55:46,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:55:46,034 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:55:46,035 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:55:46,259 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:55:47,114 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:47,259 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:55:51,261 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:55:53,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:55:53,673 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:55:58,674 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:01,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:56:01,038 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:56:01,040 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:56:01,665 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:56:01,667 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:56:01,667 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:56:01,668 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:56:02,265 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:56:03,266 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:56:03,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:05,267 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:56:07,267 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:56:08,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:13,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:16,040 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:56:16,041 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:56:16,042 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:56:19,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:23,674 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:56:23,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:56:23,832 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:56:23,832 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:56:23,833 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:56:24,274 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:56:24,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:25,274 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:56:29,276 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:56:29,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:31,038 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:56:31,038 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:56:31,040 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:56:35,128 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:40,121 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:56:40,122 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:56:40,122 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:56:40,122 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:56:40,162 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:40,280 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:56:41,281 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:56:45,282 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:56:45,329 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:46,036 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:56:46,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:56:46,037 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:56:51,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:53,675 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:56:56,676 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:56:57,839 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:56:57,840 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:56:57,841 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:56:57,841 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:56:58,287 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:56:59,288 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:57:01,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:57:01,038 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:57:01,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:57:02,099 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:03,289 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:57:07,099 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:12,100 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:16,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:57:16,038 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:57:16,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:57:18,068 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:19,697 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:57:19,698 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:57:19,698 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:57:19,699 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:57:20,296 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:57:21,296 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:57:23,297 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:57:23,676 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:57:23,677 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:25,298 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:57:28,677 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:31,039 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:57:31,040 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:57:31,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:57:34,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:35,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:57:35,578 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:57:35,578 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:57:35,579 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:57:36,302 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:57:37,303 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:57:39,303 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:57:39,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:44,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:46,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:57:46,038 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:57:46,038 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:57:50,127 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:53,022 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:57:53,023 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:57:53,024 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:57:53,024 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:57:53,309 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:57:53,676 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:57:55,309 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:57:55,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:57:57,310 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:58:01,073 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:58:01,074 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:58:01,075 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:58:01,075 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:06,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:10,162 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:58:10,163 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:58:10,163 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:58:10,164 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:58:10,315 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:58:11,203 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:11,315 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:58:15,317 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:58:16,047 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:58:16,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:58:16,053 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:58:17,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:22,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:23,677 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:58:27,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:31,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:58:31,053 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:58:31,053 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:58:31,094 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:58:31,123 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:58:31,123 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:58:31,124 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:58:31,323 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:58:33,124 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:33,323 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:58:35,324 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:58:38,559 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:43,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:46,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:58:46,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:58:46,053 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:58:48,620 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:58:48,621 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:58:48,621 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:58:48,621 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:48,622 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:58:49,329 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:58:49,329 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:58:53,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:58:53,679 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:58:53,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:58:58,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:01,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:59:01,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:59:01,053 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:59:04,157 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:09,158 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:09,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:59:09,392 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:59:09,392 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:59:09,393 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:59:10,338 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:59:11,338 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:59:13,339 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:59:14,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:16,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:59:16,053 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:59:16,053 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:59:20,142 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:23,680 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:59:25,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:28,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:59:28,053 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:59:28,053 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:59:28,054 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:59:28,344 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:59:29,345 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:59:31,063 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:59:31,064 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:59:31,064 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:59:31,066 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:33,347 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:59:36,280 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:41,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:46,053 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 06:59:46,061 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 06:59:46,061 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 06:59:47,053 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 06:59:47,054 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 06:59:47,054 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 06:59:47,055 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:47,055 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 06:59:47,352 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 06:59:49,353 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:59:51,353 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 06:59:52,431 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 06:59:53,680 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 06:59:57,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:01,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:00:01,062 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:00:01,062 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:00:03,167 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:04,563 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:00:04,564 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:00:04,564 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:00:04,565 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:00:05,359 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:00:05,359 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:00:08,788 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:09,360 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:00:13,789 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:16,055 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:00:16,062 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:00:16,062 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:00:19,121 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:21,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:00:21,044 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:00:21,044 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:00:21,045 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:00:21,365 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:00:21,365 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:00:23,681 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:00:24,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:25,367 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:00:29,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:31,055 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:00:31,062 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:00:31,062 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:00:35,095 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:40,095 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:43,525 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:00:43,526 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:00:43,526 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:00:43,527 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:00:44,373 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:00:45,374 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:00:45,566 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:46,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:00:46,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:00:46,195 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:00:47,375 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:00:51,042 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:00:53,683 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:00:56,684 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:01,195 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:01:01,196 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:01:01,199 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:01:02,325 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:03,758 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:01:03,759 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:01:03,760 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:01:03,760 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:01:04,381 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:01:05,382 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:01:07,382 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:01:07,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:12,906 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:16,191 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:01:16,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:01:16,193 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:01:18,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:23,280 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:23,684 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:01:28,685 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:29,261 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:01:29,263 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:01:29,263 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:01:29,263 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:01:29,390 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:01:31,391 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:01:31,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:01:31,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:01:31,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:01:33,392 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:01:34,598 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:39,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:44,600 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:46,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:01:46,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:01:46,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:01:49,956 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:53,685 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:01:54,630 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:01:54,631 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:01:54,631 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:01:54,631 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:01:55,401 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:01:55,402 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:01:55,671 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:01:59,403 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:02:01,126 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:01,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:02:01,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:02:01,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:02:06,970 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:11,971 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:16,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:02:16,862 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:02:16,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:02:17,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:19,022 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:02:19,023 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:02:19,023 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:02:19,024 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:02:19,410 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:02:19,410 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:02:23,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:23,412 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:02:23,686 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:02:28,687 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:31,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:02:31,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:02:31,862 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:02:33,979 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:38,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:41,442 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:02:41,443 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:02:41,444 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:02:41,444 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:02:42,418 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:02:43,419 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:02:44,483 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:45,419 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:02:46,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:02:46,862 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:02:46,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:02:49,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:02:53,687 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:02:55,688 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:00,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:01,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:03:01,862 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:03:01,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:03:04,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:03:04,535 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:03:04,535 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:03:04,535 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:03:05,427 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:03:05,427 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:03:06,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:09,429 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:03:11,725 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:16,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:16,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:03:16,864 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:03:16,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:03:18,466 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:03:18,467 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:03:18,468 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:03:18,468 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:03:19,433 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:03:19,434 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:03:22,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:23,435 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:03:23,688 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:03:27,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:31,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:03:31,865 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:03:31,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:03:32,930 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:36,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:03:36,052 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:03:36,052 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:03:36,053 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:03:36,440 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:03:37,440 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:03:38,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:39,441 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:03:43,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:46,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:03:46,862 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:03:46,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:03:48,963 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:53,694 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:03:54,695 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:03:56,843 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:03:56,844 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:03:56,844 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:03:56,844 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:03:57,447 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:03:57,448 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:03:59,885 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:01,449 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:04:01,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:04:01,864 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:04:01,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:04:04,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:09,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:13,443 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:04:13,444 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:04:13,445 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:04:13,445 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:04:13,453 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:04:15,453 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:04:15,484 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:16,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:04:16,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:04:16,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:04:17,454 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:04:20,962 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:23,690 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:04:26,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:31,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:31,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:04:31,864 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:04:31,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:04:36,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:37,442 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:04:37,443 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:04:37,444 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:04:37,446 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:04:37,461 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:04:39,461 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:04:41,462 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:04:42,632 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:46,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:04:46,862 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:04:46,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:04:47,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:52,955 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:04:53,690 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:04:58,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:01,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:05:01,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:05:01,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:05:02,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:05:02,623 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:05:02,623 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:05:02,623 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:05:03,470 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:05:04,470 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:05:04,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:08,471 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:05:09,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:14,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:16,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:05:16,866 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:05:16,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:05:19,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:23,691 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:05:24,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:05:24,344 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:05:24,345 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:05:24,345 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:05:24,477 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:05:25,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:26,478 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:05:30,479 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:05:30,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:31,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:05:31,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:05:31,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:05:35,915 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:40,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:41,090 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:05:41,091 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:05:41,091 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:05:41,093 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:05:41,483 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:05:44,485 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:05:46,266 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:46,486 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:05:46,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:05:46,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:05:46,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:05:51,982 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:05:53,693 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:05:57,694 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:01,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:06:01,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:06:01,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:06:02,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:03,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:06:03,645 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:06:03,646 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:06:03,646 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:06:04,492 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:06:06,493 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:06:08,494 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:06:08,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:13,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:16,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:06:16,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:06:16,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:06:18,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:21,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:06:21,288 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:06:21,288 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:06:21,290 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:06:21,498 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:06:23,693 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:06:24,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:24,500 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:06:26,500 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:06:29,486 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:31,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:06:31,863 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:06:31,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:06:34,987 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:39,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:06:39,553 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:06:39,553 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:06:39,554 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:06:40,505 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:06:40,593 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:42,506 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:06:44,506 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:06:45,763 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:06:46,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: stop_status +2024-04-11 07:06:46,864 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: stop_status +2024-04-11 07:06:46,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:06:51,874 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:06:56,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:01,296 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:07:01,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:07:03,105 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:07:03,106 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:03,106 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:07:03,108 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:07:03,109 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:07:03,514 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:07:04,514 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:07:06,515 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:07:07,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:08,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:12,003 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:13,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:16,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:07:18,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:19,208 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:07:19,209 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:07:19,210 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:07:19,210 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:07:19,519 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:07:22,521 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:07:22,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:23,696 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:07:23,696 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:24,521 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:07:27,796 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:28,697 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:31,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:07:33,698 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:37,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:38,699 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:41,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:07:41,954 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:07:41,954 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:07:41,956 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:07:42,527 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:07:42,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:43,987 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:44,528 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:07:46,529 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:07:46,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:07:49,137 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:52,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:53,697 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:07:54,697 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:07:57,553 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:07:58,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:07:58,792 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:07:58,792 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:07:58,792 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:07:59,534 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:07:59,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:00,535 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:08:01,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:08:04,536 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:08:04,982 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:07,554 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:08:09,983 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:12,561 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:08:14,983 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:16,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:08:17,306 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:08:17,307 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:08:17,307 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:08:17,307 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:08:17,540 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:08:20,338 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:20,542 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:08:22,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:08:22,542 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:08:23,697 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:08:25,699 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:27,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:08:30,700 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:31,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:08:35,700 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:36,962 WARNING FileStreamThread:1091 [file_stream.py:request_with_retry():687] requests_with_retry encountered retryable exception: 500 Server Error: Internal Server Error for url: https://api.wandb.ai/files/ducminh-ml71/huggingface/rimyh9a1/file_stream. func: functools.partial(>, timeout=180.0), args: ('https://api.wandb.ai/files/ducminh-ml71/huggingface/rimyh9a1/file_stream',), kwargs: {'json': {'files': {'wandb-summary.json': {'offset': 0, 'content': ['{"train/loss": 3.3958, "train/grad_norm": 99.48656463623047, "train/learning_rate": 1.2869649049440873e-05, "train/epoch": 0.05, "train/global_step": 11180, "_timestamp": 1712819261.9524338, "_runtime": 10819.103787899017, "_step": 564}']}, 'wandb-history.jsonl': {'offset': 562, 'content': ['{"train/loss": 3.2126, "train/grad_norm": 58.60544967651367, "train/learning_rate": 1.2965870820261672e-05, "train/epoch": 0.05, "train/global_step": 11140, "_timestamp": 1712819221.2956657, "_runtime": 10778.447019815445, "_step": 562}', '{"train/loss": 4.4423, "train/grad_norm": 15.332870483398438, "train/learning_rate": 1.2917749013937503e-05, "train/epoch": 0.05, "train/global_step": 11160, "_timestamp": 1712819239.2079432, "_runtime": 10796.359297275543, "_step": 563}', '{"train/loss": 3.3958, "train/grad_norm": 99.48656463623047, "train/learning_rate": 1.2869649049440873e-05, "train/epoch": 0.05, "train/global_step": 11180, "_timestamp": 1712819261.9524338, "_runtime": 10819.103787899017, "_step": 564}']}, 'wandb-events.jsonl': {'offset': 358, 'content': ['{"system.network.sent": 676280115.93, "system.network.recv": 452830156.0, "system.proc.memory.availableMB": 28276.28, "system.memory": 11.95, "system.proc.memory.rssMB": 2828.42, "system.proc.memory.percent": 8.81, "system.cpu": 28.48, "system.cpu.0.cpu_percent": 25.15, "system.cpu.1.cpu_percent": 61.31, "system.cpu.2.cpu_percent": 15.83, "system.cpu.3.cpu_percent": 21.72, "system.proc.cpu.threads": 43, "system.disk.\\\\.usagePercent": 69.0, "system.disk.\\\\.usageGB": 5566.96, "system.disk.in": 229.0, "system.disk.out": 976402.3, "system.gpu.0.memoryAllocated": 84.13, "system.gpu.1.memoryAllocated": 28.55, "system.gpu.0.memoryAllocatedBytes": 13550682112.0, "system.gpu.1.memoryAllocatedBytes": 4597940224.0, "system.gpu.0.memory": 33.13, "system.gpu.1.memory": 1.93, "system.gpu.0.gpu": 59.8, "system.gpu.1.gpu": 21.33, "system.gpu.0.temp": 76.93, "system.gpu.1.temp": 55.6, "system.gpu.0.powerWatts": 62.3, "system.gpu.1.powerWatts": 33.69, "system.gpu.0.powerPercent": 89.55, "system.gpu.1.powerPercent": 47.98, "_wandb": true, "_timestamp": 1712819243.694884, "_runtime": 10800.846238}']}, 'output.log': {'offset': 3380, 'content': ['ERROR 2024-04-11T07:07:03.631961 /opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\n', 'ERROR 2024-04-11T07:07:03.631961 warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.")\n', 'ERROR 2024-04-11T07:07:05.636647 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n', 'ERROR 2024-04-11T07:07:05.636647 warnings.warn(\n', 'ERROR 2024-04-11T07:07:05.636647 /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n', "ERROR 2024-04-11T07:07:05.636647 warnings.warn('Was asked to gather along dimension 0, but all '\n", 'ERROR 2024-04-11T07:07:21.656253 /opt/conda/lib/python3.10/site-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\n', 'ERROR 2024-04-11T07:07:21.656253 warnings.warn("Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.")\n', 'ERROR 2024-04-11T07:07:23.661960 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n', 'ERROR 2024-04-11T07:07:23.661960 warnings.warn(\n', 'ERROR 2024-04-11T07:07:23.661960 /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n', "ERROR 2024-04-11T07:07:23.661960 warnings.warn('Was asked to gather along dimension 0, but all '\n"]}}, 'dropped': 0}} +2024-04-11 07:08:37,097 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:08:38,649 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:08:38,651 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:08:38,651 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:08:38,651 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:08:39,549 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:08:40,549 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:08:41,683 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:42,361 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:08:44,551 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:08:46,846 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:46,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:08:51,847 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:52,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:08:53,698 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:08:57,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:08:57,699 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:08:57,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:08:57,896 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:08:57,896 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:08:57,896 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:08:58,556 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:09:00,557 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:09:01,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:09:02,558 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:09:03,302 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:07,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:09:08,303 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:12,623 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:09:13,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:16,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:09:18,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:18,682 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:09:18,683 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:09:18,683 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:09:18,684 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:09:19,564 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:09:20,564 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:09:22,401 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:09:23,699 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:09:23,700 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:24,566 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:09:27,402 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:09:28,701 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:31,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:09:33,702 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:37,406 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:09:38,702 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:40,761 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:09:40,762 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:09:40,763 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:09:40,763 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:09:41,572 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:09:42,503 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:09:42,573 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:09:43,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:46,574 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:09:46,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:09:49,167 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:52,510 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:09:53,701 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:09:54,701 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:09:57,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:09:59,702 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:01,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:10:03,509 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:10:03,510 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:10:03,511 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:10:03,512 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:10:03,581 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:10:05,550 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:06,582 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:10:07,243 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:10:08,583 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:10:10,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:12,249 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:10:15,921 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:16,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:10:20,922 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:22,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:10:23,702 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:10:26,321 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:10:26,322 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:10:26,323 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:10:26,323 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:26,324 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:10:26,590 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:10:28,066 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:10:28,591 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:10:30,592 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:10:31,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:31,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:10:32,592 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:10:36,623 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:37,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:10:41,624 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:42,123 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:10:46,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:46,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:10:47,469 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:10:47,470 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:10:47,470 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:10:47,472 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:10:47,598 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:10:50,600 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:10:51,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:52,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:10:52,601 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:10:53,703 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:10:56,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:10:57,189 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:01,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:01,868 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:11:06,705 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:07,191 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:07,396 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:11:07,397 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:11:07,398 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:11:07,398 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:11:07,607 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:11:08,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:11:11,899 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:12,610 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:11:12,979 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:16,868 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:11:16,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:21,901 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:21,981 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:23,703 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:11:25,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:11:25,288 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:11:25,288 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:11:25,289 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:11:25,615 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:11:26,615 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:11:27,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:27,321 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:30,617 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:11:31,868 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:11:32,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:37,015 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:37,469 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:42,151 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:42,470 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:44,727 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:11:44,729 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:11:44,729 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:11:44,729 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:11:45,622 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:11:46,622 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:11:47,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:47,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:11:47,760 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:50,624 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:11:52,491 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:52,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:53,704 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:11:57,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:11:58,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:11:59,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:11:59,612 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:11:59,612 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:11:59,613 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:11:59,627 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:12:02,374 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:12:02,629 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:12:03,859 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:04,630 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:12:08,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:12:08,860 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:13,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:12:13,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:17,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:12:17,559 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:12:17,560 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:12:17,561 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:12:17,561 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:12:17,634 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:12:18,635 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:12:19,600 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:22,637 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:12:23,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:12:23,705 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:12:24,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:28,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:12:29,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:32,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:12:34,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:38,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:12:38,332 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:12:38,334 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:12:38,334 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:12:38,335 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:12:38,643 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:12:40,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:40,644 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:12:42,645 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:12:44,056 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:12:45,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:47,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:12:50,503 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:53,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:12:53,706 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:12:55,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:12:56,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:12:56,627 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:12:56,628 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:12:56,628 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:12:56,649 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:12:58,332 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:12:58,650 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:00,651 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:00,784 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:02,376 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:13:02,652 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:05,785 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:08,347 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:13:10,785 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:11,507 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:13:11,508 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:13:11,509 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:13:11,509 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:13:11,655 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:13:12,656 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:14,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:13:16,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:16,657 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:17,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:13:21,655 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:23,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:13:23,707 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:13:26,708 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:27,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:13:27,904 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:13:27,905 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:13:27,905 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:13:28,111 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:13:28,662 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:13:30,663 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:32,055 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:32,371 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:13:32,664 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:37,056 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:37,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:13:42,056 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:42,761 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:13:45,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:13:45,435 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:13:45,436 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:13:45,436 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:13:45,670 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:13:46,670 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:47,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:48,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:13:48,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:13:50,672 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:13:52,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:53,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:13:53,708 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:13:57,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:13:58,224 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:14:02,710 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:03,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:14:05,843 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:14:05,844 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:14:05,844 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:14:05,845 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:14:06,678 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:14:07,884 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:08,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:14:08,679 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:14:10,680 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:14:13,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:13,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:14:18,070 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:18,093 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:14:23,070 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:23,606 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:14:23,709 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:14:25,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:14:25,450 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:14:25,450 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:14:25,451 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:14:25,686 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:14:26,686 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:14:28,485 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:29,166 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:14:30,688 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:14:33,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:14:33,620 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:38,179 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:14:38,621 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:43,180 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:14:43,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:14:43,532 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:14:43,532 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:14:43,533 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:14:43,693 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:14:44,565 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:44,693 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:14:48,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:14:48,695 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:14:49,728 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:53,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:14:53,710 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:14:55,711 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:14:58,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:00,712 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:01,289 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:15:01,290 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:15:01,290 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:15:01,291 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:15:01,701 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:15:02,701 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:15:03,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:03,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:15:05,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:06,703 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:15:09,011 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:10,714 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:14,018 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:15,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:17,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:15:17,727 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:15:17,727 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:15:17,728 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:15:18,707 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:15:18,707 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:15:19,427 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:19,427 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:19,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:15:20,759 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:22,708 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:15:23,711 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:15:24,431 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:26,712 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:29,432 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:31,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:34,427 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:15:36,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:37,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:15:37,658 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:15:37,658 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:15:37,658 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:15:37,715 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:15:38,715 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:15:40,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:41,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:42,717 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:15:45,378 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:46,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:49,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:15:51,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:15:53,712 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:15:55,380 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:15:55,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:15:55,639 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:15:55,639 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:15:55,639 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:15:55,721 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:15:56,722 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:15:57,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:00,381 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:00,723 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:16:02,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:04,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:16:07,809 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:10,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:12,810 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:15,389 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:16,165 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:16:16,166 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:16:16,167 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:16:16,167 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:16:16,729 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:16:18,202 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:18,730 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:16:19,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:16:20,731 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:16:23,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:23,713 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:16:24,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:28,714 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:29,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:30,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:16:30,016 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:16:30,016 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:16:30,016 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:16:30,734 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:16:32,735 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:16:34,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:34,429 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:16:34,736 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:16:39,161 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:39,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:44,162 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:44,792 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:47,261 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:16:47,263 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:16:47,263 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:16:47,263 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:16:47,741 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:16:48,742 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:16:49,295 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:49,844 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:49,845 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:16:52,743 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:16:53,713 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:16:54,714 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:16:54,973 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:16:59,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:00,120 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:03,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:17:03,830 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:17:03,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:17:03,830 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:17:04,748 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:17:04,748 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:17:04,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:05,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:05,583 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:17:05,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:08,749 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:17:10,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:10,590 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:15,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:15,600 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:20,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:20,583 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:17:23,565 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:17:23,566 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:17:23,566 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:17:23,567 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:17:23,714 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:17:23,755 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:17:24,755 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:17:25,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:26,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:28,757 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:17:30,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:31,288 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:35,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:17:35,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:38,038 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:17:38,039 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:17:38,040 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:17:38,040 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:17:38,761 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:17:38,761 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:17:40,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:41,079 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:42,762 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:17:45,772 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:46,201 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:50,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:17:51,202 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:53,715 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:17:55,774 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:17:56,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:17:57,146 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:17:57,147 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:17:57,148 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:17:57,148 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:17:57,768 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:17:58,769 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:18:00,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:02,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:02,770 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:18:05,589 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:18:07,538 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:10,876 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:12,539 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:15,885 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:17,540 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:19,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:18:19,614 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:18:19,614 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:18:19,614 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:18:19,776 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:18:20,777 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:18:21,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:21,332 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:21,332 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:18:22,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:23,716 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:18:24,778 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:18:26,337 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:27,718 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:31,338 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:32,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:35,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:18:35,948 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:18:35,948 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:18:35,948 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:18:36,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:18:36,783 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:18:36,783 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:18:37,983 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:40,785 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:18:41,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:43,113 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:46,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:48,114 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:50,557 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:18:50,558 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:18:50,559 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:18:50,559 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:18:50,788 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:18:51,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:18:52,789 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:18:53,594 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:18:53,722 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:18:54,790 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:18:57,292 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:18:58,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:02,293 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:19:03,742 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:06,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:19:08,742 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:10,842 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:19:10,844 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:19:10,844 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:19:10,844 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:19:11,796 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:19:12,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:19:12,797 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:19:13,884 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:14,798 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:19:17,566 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:19:19,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:21,332 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:19:23,718 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:19:24,718 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:26,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:19:27,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:19:27,495 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:19:27,496 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:19:27,497 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:19:27,802 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:19:28,803 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:19:30,528 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:32,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:19:32,805 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:19:35,699 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:36,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:19:40,700 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:41,101 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:19:41,102 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:19:41,103 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:19:41,103 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:19:41,808 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:19:42,809 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:19:42,818 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:19:44,810 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:19:46,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:47,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:19:51,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:51,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:19:53,719 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:19:56,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:19:56,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:01,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:20:01,519 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:20:01,519 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:20:01,521 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:20:01,816 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:20:01,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:02,551 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:02,817 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:20:06,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:20:06,818 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:20:07,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:12,262 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:12,710 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:17,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:17,711 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:21,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:20:22,108 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:20:22,109 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:20:22,109 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:20:22,110 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:20:22,824 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:20:22,824 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:20:23,149 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:23,720 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:20:26,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:26,826 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:20:28,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:31,825 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:33,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:36,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:20:38,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:39,132 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:20:39,133 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:20:39,133 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:20:39,135 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:20:39,835 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:20:40,835 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:20:41,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:42,836 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:20:43,329 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:46,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:48,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:51,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:20:53,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:20:53,721 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:20:56,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:20:58,723 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:00,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:21:00,111 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:21:00,111 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:21:00,112 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:21:00,843 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:21:00,844 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:21:01,876 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:21:04,308 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:04,845 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:21:06,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:21:09,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:11,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:21:14,310 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:16,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:21:16,995 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:21:16,996 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:21:16,996 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:21:16,998 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:21:17,851 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:21:18,851 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:21:20,028 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:20,852 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:21:21,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:21:23,722 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:21:25,723 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:27,038 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:21:30,724 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:32,039 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:21:35,725 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:36,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:21:38,433 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:21:38,434 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:21:38,434 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:21:38,435 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:21:38,859 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:21:40,860 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:21:41,475 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:42,168 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:21:42,860 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:21:46,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:47,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:21:51,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:21:51,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:21:53,723 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:21:55,460 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:21:55,461 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:21:55,461 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:21:55,462 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:21:55,865 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:21:56,866 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:21:57,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:21:57,502 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:00,868 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:22:02,202 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:22:02,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:06,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:22:07,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:12,208 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:22:12,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:14,485 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:22:14,486 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:22:14,486 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:22:14,487 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:22:14,873 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:22:16,874 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:22:17,214 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:22:18,634 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:18,875 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:22:21,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:22:23,635 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:23,724 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:22:27,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:22:28,725 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:32,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:22:32,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:22:32,942 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:22:32,942 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:22:32,943 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:22:33,882 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:22:33,974 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:34,882 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:22:36,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:22:36,886 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:22:39,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:41,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:22:44,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:46,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:22:49,087 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:51,348 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:22:53,725 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:22:54,725 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:22:56,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:22:57,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:22:57,092 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:22:57,092 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:22:57,092 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:22:57,895 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:22:59,895 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:23:00,101 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:01,804 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:23:01,896 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:23:05,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:06,348 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:23:10,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:10,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:23:10,742 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:23:10,742 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:23:10,743 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:23:10,899 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:23:12,485 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:23:13,900 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:23:15,901 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:23:15,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:17,491 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:23:20,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:21,348 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:23:23,725 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:23:26,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:23:26,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:31,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:23:31,727 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:32,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:23:32,520 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:23:32,520 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:23:32,520 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:23:32,907 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:23:33,908 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:23:36,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:23:37,698 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:37,909 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:23:42,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:23:42,699 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:47,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:23:47,700 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:47,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:23:47,799 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:23:47,799 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:23:47,799 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:23:47,913 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:23:49,914 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:23:51,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:23:52,990 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:23:53,915 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:23:55,332 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:23:56,525 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:23:58,333 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:01,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:24:02,775 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:24:02,777 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:24:02,777 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:24:02,777 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:24:02,919 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:24:03,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:05,920 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:24:06,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:24:07,921 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:24:08,926 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:11,528 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:24:13,927 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:16,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:24:18,927 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:21,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:24:22,832 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:24:22,833 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:24:22,833 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:24:22,834 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:24:22,927 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:24:23,727 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:24:24,728 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:25,928 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:24:26,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:24:27,929 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:24:30,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:31,555 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:24:35,237 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:36,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:24:40,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:41,567 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:24:42,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:24:42,653 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:24:42,653 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:24:42,653 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:24:42,934 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:24:43,935 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:24:45,682 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:47,376 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:24:47,936 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:24:50,827 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:24:51,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:24:53,728 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:24:56,378 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:24:56,729 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:01,379 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:25:01,731 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:02,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:25:02,600 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:25:02,600 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:25:02,602 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:25:02,942 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:25:03,942 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:25:06,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:25:06,753 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:07,944 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:25:11,753 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:12,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:25:16,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:17,189 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:25:21,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:25:21,755 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:22,132 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:25:22,133 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:25:22,134 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:25:22,134 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:25:22,949 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:25:23,729 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:25:23,950 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:25:26,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:25:27,525 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:27,951 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:25:31,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:25:32,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:36,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:25:37,527 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:41,859 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:25:41,906 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:25:41,907 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:25:41,908 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:25:41,910 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:25:41,956 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:25:42,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:43,956 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:25:47,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:25:47,958 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:25:48,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:51,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:25:53,053 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:25:53,730 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:25:56,633 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:25:58,731 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:01,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:26:03,393 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:26:03,394 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:26:03,394 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:26:03,394 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:26:03,964 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:26:04,426 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:05,965 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:26:06,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:26:07,966 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:26:09,572 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:12,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:26:14,573 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:17,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:26:19,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:21,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:26:23,731 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:26:24,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:26:24,207 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:26:24,207 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:26:24,208 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:26:24,972 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:26:25,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:25,972 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:26:26,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:26:29,974 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:26:30,401 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:31,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:26:35,402 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:36,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:26:40,403 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:41,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:26:44,336 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:26:44,337 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:26:44,338 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:26:44,338 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:26:44,979 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:26:45,979 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:26:46,373 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:47,093 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:26:49,981 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:26:51,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:26:51,566 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:53,732 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:26:56,733 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:26:57,106 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:01,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:02,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:27:02,081 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:27:02,081 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:27:02,082 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:27:02,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:02,986 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:27:03,986 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:27:06,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:27:07,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:07,988 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:27:11,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:12,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:16,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:17,292 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:21,023 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:27:21,024 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:27:21,024 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:27:21,025 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:27:21,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:27:21,992 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:27:23,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:23,733 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:27:23,993 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:27:25,994 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:27:26,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:28,148 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:31,600 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:33,149 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:36,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:27:38,150 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:41,606 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:42,527 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:27:42,528 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:27:42,529 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:27:42,529 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:27:43,000 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:27:43,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:44,000 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:27:47,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:48,002 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:27:48,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:51,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:27:53,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:27:53,733 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:27:57,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:27:58,735 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:02,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:28:03,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:05,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:28:05,932 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:28:05,933 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:28:05,933 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:28:06,008 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:28:06,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:28:08,009 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:28:08,965 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:11,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:28:12,011 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:28:14,145 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:16,690 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:28:19,146 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:21,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:28:23,735 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:28:24,735 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:26,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:28:26,598 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:28:26,598 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:28:26,599 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:28:26,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:28:27,016 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:28:28,016 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:28:30,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:32,018 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:28:32,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:28:35,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:36,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:28:40,721 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:42,324 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:28:45,114 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:28:45,116 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:28:45,116 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:28:45,116 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:28:46,024 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:28:46,152 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:47,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:28:48,024 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:28:50,025 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:28:51,306 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:51,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:28:53,736 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:28:56,737 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:28:56,884 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:01,737 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:01,890 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:04,878 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:29:04,879 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:29:04,880 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:29:04,880 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:29:05,030 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:29:06,031 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:29:06,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:29:06,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:10,032 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:29:11,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:12,041 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:16,619 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:17,041 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:21,606 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:29:22,042 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:23,737 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:29:24,437 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:29:24,439 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:29:24,439 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:29:24,439 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:29:25,038 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:29:26,038 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:29:27,158 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:27,459 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:30,040 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:29:32,164 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:32,615 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:36,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:29:37,616 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:42,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:42,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:43,556 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:29:43,557 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:29:43,557 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:29:43,558 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:29:44,045 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:29:46,046 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:29:47,333 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:47,703 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:48,047 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:29:51,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:29:52,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:29:53,737 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:29:57,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:29:57,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:02,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:30:02,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:06,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:30:07,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:09,777 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:30:09,779 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:30:09,779 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:30:09,779 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:30:10,055 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:30:12,056 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:30:12,510 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:30:12,819 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:14,057 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:30:17,516 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:30:17,983 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:21,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:30:22,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:23,738 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:30:27,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:30:28,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:30,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:30:30,284 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:30:30,284 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:30:30,285 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:30:31,063 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:30:32,063 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:30:33,012 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:30:34,714 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:36,065 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:30:36,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:30:39,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:42,020 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:30:44,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:47,021 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:30:49,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:51,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:30:53,191 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:30:53,192 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:30:53,193 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:30:53,193 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:30:53,739 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:30:54,071 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:30:54,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:30:56,072 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:30:56,767 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:30:58,073 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:31:00,320 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:01,768 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:31:05,321 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:06,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:31:10,322 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:10,516 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:31:10,517 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:31:10,517 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:31:10,517 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:31:11,077 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:31:12,078 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:31:12,259 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:31:15,961 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:16,083 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:31:17,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:31:20,962 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:21,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:31:23,740 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:31:26,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:27,271 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:31:30,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:31:30,827 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:31:30,827 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:31:30,828 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:31:31,088 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:31:31,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:32,089 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:31:32,528 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:31:36,090 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:31:36,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:31:36,951 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:41,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:42,535 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:31:46,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:47,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:31:51,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:31:51,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:52,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:31:52,337 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:31:52,337 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:31:52,337 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:31:53,096 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:31:53,741 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:31:54,097 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:31:57,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:31:57,453 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:31:58,098 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:32:02,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:32:02,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:06,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:32:07,455 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:12,050 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:32:12,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:15,312 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:32:15,313 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:32:15,313 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:32:15,314 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:32:16,105 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:32:16,105 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:32:17,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:32:18,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:20,106 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:32:21,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:32:23,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:23,742 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:32:27,070 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:32:28,743 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:29,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:32:29,474 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:32:29,475 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:32:31,337 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:32:32,071 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:32:32,111 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:32:32,111 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:32:34,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:36,112 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:32:36,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:32:39,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:42,298 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:32:44,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:47,299 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:32:48,718 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:32:48,719 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:32:48,719 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:32:48,720 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:32:49,117 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:32:49,751 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:50,117 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:32:51,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:32:53,742 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:32:54,119 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:32:55,743 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:32:57,453 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:00,744 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:02,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:05,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:06,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:33:08,973 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:33:08,975 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:33:08,975 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:33:08,975 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:33:09,124 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:33:10,125 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:33:11,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:11,704 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:14,126 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:33:16,133 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:16,710 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:21,134 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:21,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:33:23,744 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:33:26,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:26,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:29,244 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:33:29,245 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:33:29,246 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:33:29,246 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:33:30,133 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:33:30,134 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:33:31,974 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:32,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:34,135 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:33:36,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:33:37,417 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:41,987 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:42,419 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:46,989 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:47,420 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:51,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:33:51,344 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:33:51,345 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:33:51,345 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:33:51,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:33:52,142 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:33:52,143 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:33:53,377 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:33:53,744 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:33:56,144 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:33:57,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:33:58,478 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:02,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:34:03,479 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:06,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:34:07,977 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:34:07,979 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:34:07,979 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:34:07,979 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:34:08,148 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:34:09,018 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:10,149 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:34:11,699 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:34:12,149 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:34:14,161 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:16,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:34:19,162 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:21,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:34:22,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:34:22,270 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:34:22,270 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:34:22,283 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:34:23,154 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:34:24,154 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:34:24,310 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:25,833 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:34:27,005 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:34:28,156 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:34:29,834 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:32,006 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:34:34,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:36,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:34:39,768 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:34:39,769 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:34:39,769 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:34:39,770 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:34:40,161 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:34:40,809 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:42,162 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:34:42,485 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:34:44,163 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:34:45,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:47,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:34:50,901 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:51,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:34:53,746 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:34:56,747 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:34:56,837 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:34:56,838 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:34:56,838 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:34:56,840 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:34:57,168 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:34:57,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:34:58,168 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:35:02,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:02,170 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:35:02,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:35:06,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:35:07,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:12,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:12,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:35:15,388 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:35:15,389 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:35:15,389 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:35:15,390 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:35:16,175 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:35:16,176 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:35:17,421 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:18,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:35:20,177 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:35:21,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:35:22,784 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:23,747 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:35:27,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:35:28,748 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:32,361 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:35:33,749 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:34,381 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:35:34,382 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:35:34,382 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:35:34,384 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:35:35,183 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:35:36,183 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:35:36,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:35:38,184 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:35:39,524 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:42,118 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:35:44,525 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:47,121 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:35:49,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:51,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:35:52,132 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:35:52,133 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:35:52,134 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:35:52,134 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:35:52,190 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:35:53,748 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:35:54,191 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:35:54,749 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:35:56,191 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:35:57,906 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:00,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:02,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:05,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:06,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:36:06,817 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:36:06,818 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:36:06,820 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:36:06,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:36:07,195 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:36:08,196 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:36:11,333 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:12,198 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:36:12,596 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:16,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:17,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:21,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:21,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:36:23,749 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:36:26,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:36:26,365 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:36:26,365 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:36:26,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:26,367 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:36:27,203 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:36:28,146 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:28,204 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:36:30,205 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:36:31,727 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:33,153 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:36,728 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:36,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:36:41,729 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:42,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:44,760 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:36:44,761 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:36:44,761 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:36:44,763 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:36:45,211 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:36:46,211 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:36:46,796 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:47,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:50,213 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:36:51,964 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:51,988 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:36:53,751 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:36:57,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:36:57,752 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:36:58,633 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:36:58,635 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:36:58,635 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:36:58,635 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:36:59,217 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:37:00,218 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:37:03,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:03,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:37:04,219 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:37:06,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:37:08,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:12,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:37:13,111 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:15,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:37:15,499 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:37:15,499 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:37:15,501 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:37:16,224 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:37:16,224 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:37:18,106 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:37:18,532 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:20,225 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:37:21,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:37:23,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:23,751 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:37:27,108 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:37:28,753 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:32,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:37:33,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:34,756 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:37:34,758 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:37:34,758 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:37:34,759 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:37:35,230 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:37:36,231 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:37:37,372 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:37:37,373 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:37:38,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:40,232 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:37:42,500 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:37:43,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:47,508 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:37:48,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:51,794 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:37:51,795 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:37:51,796 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:37:51,798 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:37:52,237 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:37:52,237 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:37:52,372 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:37:53,753 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:37:54,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:37:56,238 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:37:57,513 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:00,000 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:02,517 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:05,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:07,373 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:38:10,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:11,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:38:11,352 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:38:11,353 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:38:11,353 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:38:12,244 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:38:12,244 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:38:13,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:15,542 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:16,245 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:38:18,079 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:20,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:22,396 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:38:23,754 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:38:25,755 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:26,097 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:38:26,099 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:38:26,099 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:38:26,099 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:38:26,249 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:38:27,804 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:28,250 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:38:30,250 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:38:31,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:32,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:36,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:37,377 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:38:41,329 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:42,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:43,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:38:43,954 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:38:43,954 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:38:43,954 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:38:44,255 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:38:46,257 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:38:46,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:48,257 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:38:48,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:52,178 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:38:52,377 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:38:53,755 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:38:57,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:38:57,756 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:01,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:39:01,341 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:39:01,341 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:39:01,342 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:39:02,263 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:39:02,263 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:39:03,049 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:39:03,373 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:06,264 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:39:07,395 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:39:08,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:13,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:39:13,524 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:17,145 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:39:17,146 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:39:17,146 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:39:17,147 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:39:17,269 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:39:18,269 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:39:18,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:39:19,179 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:22,271 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:39:22,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:39:23,756 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:39:24,756 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:27,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:39:29,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:32,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:39:33,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:39:33,870 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:39:33,870 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:39:33,871 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:39:34,276 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:39:34,276 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:39:34,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:37,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:39:40,278 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:39:40,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:42,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:39:45,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:47,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:39:50,827 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:52,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:39:52,749 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:39:52,750 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:39:52,750 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:39:52,751 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:39:53,283 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:39:53,757 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:39:54,283 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:39:55,923 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:39:56,284 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:39:58,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:00,924 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:03,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:05,928 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:07,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:40:09,128 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:40:09,130 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:40:09,130 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:40:09,130 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:40:09,288 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:40:10,290 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:40:11,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:12,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:15,292 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:40:16,292 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:17,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:21,293 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:22,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:40:23,758 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:40:25,178 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:40:25,180 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:40:25,180 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:40:25,181 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:40:25,296 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:40:27,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:27,297 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:40:27,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:31,299 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:40:32,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:32,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:37,388 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:40:37,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:40,876 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:40:40,878 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:40:40,878 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:40:40,878 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:40:41,303 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:40:42,615 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:42,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:43,303 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:40:45,304 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:40:47,305 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:40:47,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:48,122 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:52,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:40:53,123 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:53,759 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:40:57,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:40:57,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:40:57,680 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:40:57,680 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:40:57,680 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:40:58,309 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:40:58,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:40:59,309 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:41:03,311 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:41:03,374 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:41:04,266 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:07,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:41:09,267 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:12,439 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:41:14,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:17,440 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:41:17,510 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:41:17,512 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:41:17,512 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:41:17,512 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:41:18,317 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:41:19,317 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:41:19,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:22,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:41:23,319 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:41:23,760 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:41:24,761 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:28,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:41:29,762 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:33,262 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:41:34,762 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:35,974 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:41:35,976 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:41:35,976 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:41:35,977 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:41:36,324 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:41:37,324 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:41:37,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:41:40,410 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:41,326 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:41:42,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:41:45,411 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:47,748 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:41:50,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:52,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:41:53,495 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:41:53,496 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:41:53,496 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:41:53,497 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:41:53,762 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:41:54,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:41:55,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:41:55,762 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:41:58,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:41:59,333 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:42:01,674 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:03,217 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:42:06,675 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:07,735 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:42:11,676 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:13,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:42:14,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:42:14,904 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:42:14,904 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:42:14,904 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:42:15,339 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:42:16,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:17,339 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:42:18,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:42:19,340 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:42:22,076 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:22,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:42:23,762 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:42:27,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:28,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:42:32,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:42:32,632 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:42:32,633 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:42:32,633 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:42:33,346 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:42:33,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:34,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:42:35,347 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:42:37,347 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:42:37,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:42:38,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:43,547 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:42:43,815 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:48,556 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:42:48,815 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:52,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:42:53,763 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:42:54,186 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:42:54,187 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:42:54,187 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:42:54,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:42:54,188 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:42:54,354 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:42:55,354 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:42:57,901 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:42:59,356 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:42:59,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:02,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:43:04,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:07,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:43:09,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:12,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:43:14,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:14,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:43:14,387 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:43:14,388 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:43:14,388 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:43:15,362 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:43:17,362 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:43:18,103 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:43:19,363 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:43:19,758 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:22,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:43:23,764 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:43:24,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:28,104 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:43:29,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:33,105 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:43:34,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:36,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:43:36,311 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:43:36,311 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:43:36,311 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:43:36,370 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:43:37,370 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:43:38,018 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:43:40,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:41,372 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:43:43,896 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:43:45,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:49,121 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:43:50,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:53,018 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:43:53,765 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:43:55,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:43:57,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:43:57,667 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:43:57,667 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:43:57,667 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:43:58,123 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:43:58,378 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:43:59,378 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:44:00,878 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:03,380 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:44:03,395 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:44:05,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:08,018 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:44:10,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:13,402 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:44:15,883 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:18,404 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:44:20,884 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:23,018 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:44:23,311 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:44:23,312 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:44:23,312 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:44:23,312 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:44:23,388 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:44:23,766 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:44:25,389 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:44:26,411 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:27,389 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:44:28,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:44:29,390 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:44:31,529 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:33,054 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:44:36,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:38,019 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:44:40,078 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:44:40,079 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:44:40,079 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:44:40,080 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:44:40,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:44:41,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:44:42,120 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:43,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:44:45,397 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:44:47,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:48,817 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:44:52,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:53,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:44:53,767 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:44:54,414 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:44:54,416 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:44:54,416 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:44:54,416 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:44:55,401 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:44:55,402 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:44:57,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:44:59,152 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:44:59,403 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:45:02,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:04,153 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:45:07,657 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:08,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:45:12,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:14,168 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:45:16,965 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:45:16,966 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:45:16,966 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:45:16,967 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:45:17,412 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:45:17,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:19,413 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:45:19,694 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:45:21,414 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:45:23,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:45:23,437 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:23,767 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:45:28,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:45:28,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:33,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:45:33,770 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:38,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:45:38,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:40,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:45:40,447 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:45:40,448 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:45:40,448 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:45:41,421 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:45:41,422 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:45:44,172 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:45:44,624 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:45,423 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:45:49,174 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:45:49,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:53,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:45:53,769 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:45:54,770 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:45:58,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:45:58,369 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:45:58,369 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:45:58,370 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:45:58,428 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:45:59,176 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:45:59,428 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:46:00,409 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:03,430 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:46:05,004 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:46:05,567 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:08,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:46:10,568 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:14,010 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:46:15,569 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:19,012 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:46:20,570 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:21,782 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:46:21,783 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:46:21,784 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:46:21,784 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:46:22,437 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:46:23,438 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:46:23,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:46:23,770 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:46:26,180 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:27,440 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:46:28,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:46:31,181 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:33,510 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:46:36,182 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:38,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:46:39,964 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:46:39,966 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:46:39,966 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:46:39,966 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:46:40,445 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:46:41,446 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:46:42,006 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:43,819 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:46:45,447 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:46:47,260 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:48,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:46:52,260 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:53,504 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:46:53,770 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:46:57,486 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:46:57,487 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:46:57,487 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:46:57,487 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:46:57,488 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:46:58,452 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:46:59,190 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:46:59,453 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:47:01,454 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:47:02,932 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:03,454 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:47:04,207 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:47:07,933 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:08,504 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:47:12,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:14,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:47:17,935 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:19,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:47:19,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:47:19,330 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:47:19,330 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:47:19,330 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:47:19,461 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:47:21,462 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:47:23,463 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:47:23,504 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:47:23,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:23,772 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:47:28,773 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:29,065 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:47:33,774 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:34,072 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:47:38,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:47:38,774 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:40,786 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:47:40,787 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:47:40,788 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:47:40,788 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:47:41,470 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:47:43,471 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:47:43,529 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:47:43,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:45,472 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:47:48,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:47:49,288 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:53,539 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:47:53,773 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:47:54,773 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:47:58,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:47:59,774 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:03,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:48:04,775 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:05,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:48:05,668 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:48:05,668 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:48:05,669 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:48:06,480 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:48:07,480 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:48:08,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:48:09,481 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:48:09,833 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:11,482 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:48:14,411 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:48:14,834 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:19,413 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:48:19,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:23,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:48:23,773 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:48:24,984 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:48:24,985 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:48:24,985 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:48:24,985 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:24,986 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:48:25,487 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:48:27,488 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:48:28,717 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:48:29,489 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:48:30,189 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:33,731 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:48:35,190 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:38,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:48:39,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:48:39,291 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:48:39,291 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:48:39,292 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:48:39,494 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:48:40,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:41,495 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:48:43,495 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:48:44,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:48:45,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:49,297 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:48:50,447 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:53,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:48:53,774 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:48:54,251 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:48:54,252 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:48:54,252 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:48:54,253 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:48:54,500 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:48:55,500 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:48:56,289 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:48:59,125 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:48:59,502 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:49:01,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:04,135 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:06,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:08,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:49:11,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:14,348 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:16,694 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:19,064 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:49:19,065 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:49:19,065 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:49:19,065 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:49:19,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:19,510 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:49:21,511 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:49:22,098 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:23,511 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:49:23,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:49:23,775 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:49:27,776 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:28,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:32,777 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:33,818 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:35,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:49:35,908 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:49:35,908 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:49:35,909 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:49:36,517 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:49:37,517 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:49:37,942 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:38,515 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:49:41,519 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:49:43,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:43,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:48,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:48,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:51,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:49:51,002 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:49:51,003 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:49:51,003 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:49:51,522 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:49:53,523 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:49:53,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:53,727 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:49:53,777 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:49:53,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:49:55,524 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:49:58,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:49:59,322 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:03,800 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:50:04,323 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:08,731 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:50:09,324 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:12,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:50:12,034 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:50:12,035 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:50:12,035 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:50:12,531 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:50:13,532 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:50:13,811 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:50:15,075 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:17,534 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:50:18,817 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:50:20,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:23,731 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:50:23,778 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:50:25,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:28,825 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:50:30,780 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:31,976 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:50:31,977 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:50:31,977 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:50:31,978 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:50:32,540 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:50:33,541 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:50:34,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:50:36,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:37,542 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:50:38,731 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:50:41,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:43,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:50:46,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:48,899 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:50:50,424 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:50:50,425 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:50:50,425 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:50:50,426 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:50:50,548 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:50:51,458 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:51,548 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:50:53,731 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:50:53,779 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:50:55,550 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:50:56,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:50:59,149 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:01,784 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:04,151 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:06,207 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:51:06,208 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:51:06,208 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:51:06,209 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:51:06,554 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:51:07,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:07,554 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:51:08,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:51:11,556 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:51:12,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:13,932 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:17,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:18,933 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:22,361 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:23,780 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:51:23,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:51:24,132 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:51:24,134 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:51:24,134 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:51:24,134 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:51:24,561 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:51:25,561 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:51:27,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:29,563 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:51:29,768 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:32,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:34,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:37,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:38,790 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:51:39,827 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:51:39,828 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:51:39,829 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:51:39,829 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:51:40,567 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:51:41,568 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:51:42,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:43,568 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:51:44,568 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:48,042 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:49,569 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:53,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:53,780 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:51:53,790 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:51:54,527 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:51:54,528 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:51:54,528 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:51:54,528 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:51:54,572 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:51:55,573 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:51:58,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:51:59,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:51:59,574 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:52:03,727 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:04,277 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:08,728 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:08,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:52:11,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:52:11,455 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:52:11,456 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:52:11,579 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:52:11,763 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:52:13,580 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:52:13,764 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:14,083 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:15,581 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:52:19,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:19,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:23,782 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:52:23,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:52:24,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:28,492 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:52:28,493 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:52:28,494 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:52:28,494 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:52:28,586 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:52:29,087 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:29,586 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:52:30,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:33,588 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:52:34,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:36,005 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:38,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:52:41,006 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:44,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:46,007 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:49,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:51,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:52,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:52:52,876 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:52:52,877 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:52:52,877 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:52:53,595 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:52:53,596 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:52:53,783 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:52:54,591 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:54,591 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:52:54,592 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:52:56,020 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:52:57,597 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:52:59,598 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:01,021 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:04,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:06,022 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:09,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:09,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:53:11,023 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:12,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:53:12,872 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:53:12,873 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:53:12,873 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:53:13,604 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:53:13,604 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:53:15,604 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:16,079 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:17,606 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:53:20,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:21,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:23,784 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:53:24,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:53:26,785 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:29,624 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:30,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:53:30,673 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:53:30,674 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:53:30,674 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:53:31,611 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:53:31,611 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:53:32,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:35,395 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:35,612 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:53:37,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:39,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:53:42,868 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:45,399 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:47,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:50,402 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:53:51,509 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:53:51,510 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:53:51,510 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:53:51,511 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:53:51,618 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:53:53,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:53:53,619 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:53:53,785 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:53:54,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:53:55,620 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:53:58,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:00,225 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:54:03,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:05,227 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:54:08,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:09,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:54:11,482 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:54:11,483 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:54:11,483 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:54:11,484 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:54:11,625 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:54:13,626 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:54:14,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:15,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:54:17,628 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:54:19,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:20,232 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:54:23,785 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:54:24,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:54:25,786 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:30,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:54:30,787 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:33,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:54:33,046 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:54:33,046 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:54:33,047 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:54:33,634 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:54:33,634 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:54:35,760 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:54:36,079 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:37,635 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:54:39,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:54:41,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:44,770 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:54:46,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:49,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:54:51,292 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:53,786 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:54:54,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:54:56,787 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:54:59,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:01,462 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:55:01,463 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:55:01,463 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:55:01,463 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:55:01,643 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:55:02,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:03,644 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:55:05,185 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:05,645 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:55:07,665 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:09,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:55:12,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:15,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:17,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:20,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:20,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:55:20,856 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:55:20,856 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:55:20,856 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:55:21,650 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:55:21,651 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:55:22,888 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:23,787 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:55:24,599 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:55:25,652 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:55:28,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:30,591 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:33,044 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:35,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:36,459 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:55:36,460 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:55:36,460 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:55:36,461 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:55:36,655 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:55:37,656 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:55:38,491 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:39,600 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:55:41,657 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:55:43,677 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:45,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:48,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:50,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:52,686 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:55:52,688 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:55:52,688 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:55:52,688 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:55:53,662 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:55:53,662 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:55:53,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:55:53,788 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:55:55,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:55:55,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:55:57,663 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:55:58,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:00,495 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:56:03,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:05,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:56:06,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:56:06,473 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:56:06,474 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:56:06,474 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:56:06,666 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:56:07,667 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:56:09,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:10,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:56:11,668 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:56:14,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:16,202 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:56:19,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:21,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:56:23,789 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:56:23,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:56:23,909 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:56:23,909 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:56:23,909 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:56:24,673 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:56:24,942 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:25,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:56:25,673 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:56:27,674 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:56:30,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:30,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:56:35,093 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:35,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:56:40,094 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:40,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:56:42,508 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:56:42,510 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:56:42,510 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:56:42,510 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:56:42,680 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:56:43,680 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:56:45,539 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:45,681 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:56:46,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:56:47,682 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:56:50,683 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:51,398 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:56:53,790 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:56:55,623 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:56:55,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:56:59,991 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:56:59,992 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:56:59,992 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:56:59,992 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:57:00,688 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:57:01,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:01,688 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:57:01,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:57:03,689 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:57:06,143 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:06,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:57:10,624 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:57:11,143 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:15,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:57:16,144 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:20,733 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:57:20,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:57:20,737 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:57:20,737 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:57:20,738 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:57:21,695 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:57:21,695 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:57:21,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:23,791 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:57:25,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:57:25,697 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:57:26,950 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:31,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:57:31,950 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:36,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:57:36,436 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:57:36,437 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:57:36,437 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:57:36,479 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:57:36,701 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:57:37,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:37,701 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:57:39,702 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:57:40,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:57:42,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:46,173 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:57:47,628 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:51,174 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:57:52,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:53,792 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:57:55,145 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:57:55,146 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:57:55,146 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:57:55,146 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:57:55,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:57:55,708 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:57:55,709 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:57:58,167 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:57:59,710 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:58:00,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:58:03,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:05,868 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:58:08,282 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:10,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:58:13,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:15,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:58:16,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:58:16,029 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:58:16,029 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:58:16,030 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:58:16,717 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:58:17,718 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:58:19,060 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:19,719 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:58:21,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:58:23,793 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:58:24,794 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:25,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:58:29,796 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:30,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:58:30,958 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:58:30,959 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:58:30,959 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:58:31,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:58:31,723 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:58:32,724 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:58:35,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:36,683 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:58:36,728 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:58:40,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:40,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:58:45,171 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:45,690 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:58:50,172 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:58:50,298 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:58:50,300 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:58:50,300 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:58:50,300 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:58:50,696 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:58:50,733 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:58:52,734 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:58:53,794 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:58:54,735 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:58:55,640 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:58:55,794 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:00,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:01,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:59:05,796 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:06,033 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:59:07,491 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:59:07,492 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:59:07,493 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:59:07,493 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:59:07,740 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:59:08,740 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:59:10,641 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:59:11,750 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:12,742 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:59:16,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:59:16,751 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:21,210 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:59:21,751 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:23,437 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:59:23,438 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:59:23,439 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:59:23,439 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:59:23,748 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:59:23,794 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:59:24,746 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:59:26,061 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:59:27,604 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:28,748 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:59:31,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:59:32,604 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:36,224 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:59:37,606 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:41,020 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 07:59:41,021 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 07:59:41,022 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 07:59:41,022 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 07:59:41,060 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:59:41,752 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 07:59:42,753 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:59:43,054 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:46,744 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:59:46,754 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 07:59:48,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:51,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 07:59:53,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 07:59:53,795 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 07:59:56,100 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 07:59:58,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:01,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:00:01,532 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:00:01,532 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:00:01,533 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:00:01,748 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:00:01,760 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:00:02,760 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:00:04,571 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:06,762 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:00:07,263 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:00:09,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:11,066 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:00:14,690 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:16,265 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:00:19,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:21,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:00:23,796 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:00:24,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:26,066 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:00:26,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:00:26,959 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:00:26,960 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:00:26,960 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:00:27,769 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:00:28,770 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:00:29,990 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:31,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:00:32,771 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:00:35,108 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:36,693 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:00:40,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:41,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:00:45,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:46,695 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:00:50,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:50,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:00:50,562 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:00:50,562 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:00:50,563 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:00:50,778 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:00:52,289 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:00:52,779 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:00:53,797 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:00:54,780 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:00:55,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:00:56,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:01:00,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:01,296 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:01:05,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:06,297 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:01:10,800 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:10,978 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:01:10,979 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:01:10,980 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:01:10,980 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:01:11,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:01:11,786 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:01:12,786 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:01:16,133 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:16,723 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:01:16,788 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:01:21,134 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:21,724 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:01:23,797 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:01:26,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:01:26,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:31,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:31,801 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:01:32,496 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:01:32,497 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:01:32,497 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:01:32,498 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:01:32,794 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:01:34,795 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:01:37,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:01:37,637 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:38,796 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:01:41,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:01:42,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:46,208 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:01:47,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:50,163 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:01:50,164 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:01:50,165 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:01:50,165 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:01:50,800 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:01:51,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:01:52,801 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:01:53,200 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:01:53,799 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:01:54,802 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:01:56,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:01:58,800 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:01,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:03,801 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:06,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:06,959 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:02:06,961 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:02:06,961 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:02:06,961 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:02:07,806 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:02:08,807 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:02:08,992 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:11,069 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:02:12,808 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:02:14,104 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:16,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:19,105 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:21,674 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:23,799 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:02:24,272 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:02:24,273 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:02:24,274 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:02:24,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:24,275 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:02:24,812 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:02:26,813 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:02:26,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:26,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:02:28,815 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:02:29,455 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:31,990 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:34,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:36,991 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:39,457 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:41,509 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:02:41,511 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:02:41,511 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:02:41,511 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:02:41,820 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:02:41,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:02:42,820 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:02:44,541 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:46,822 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:02:47,223 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:49,832 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:52,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:02:53,800 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:02:55,802 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:02:56,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:02:59,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:02:59,575 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:02:59,576 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:02:59,576 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:02:59,827 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:03:01,615 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:02,347 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:03:02,828 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:04,829 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:06,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:07,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:03:11,792 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:11,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:03:16,793 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:17,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:03:17,723 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:03:17,724 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:03:17,724 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:03:17,724 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:03:17,834 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:03:20,835 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:22,196 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:22,467 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:03:22,836 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:23,801 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:03:26,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:03:27,802 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:32,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:03:32,803 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:37,542 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:03:37,804 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:38,605 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:03:38,606 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:03:38,606 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:03:38,607 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:03:38,841 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:03:40,842 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:41,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:03:42,843 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:43,760 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:44,844 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:47,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:03:48,761 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:52,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:03:53,417 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:03:53,418 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:03:53,418 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:03:53,419 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:03:53,802 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:03:53,803 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:03:53,848 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:03:54,848 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:56,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:03:58,850 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:03:59,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:02,174 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:04:04,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:07,175 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:04:09,646 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:11,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:04:12,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:04:12,798 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:04:12,799 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:04:12,799 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:04:12,855 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:04:14,838 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:14,856 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:04:17,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:04:18,858 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:04:19,966 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:22,550 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:04:23,803 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:04:25,804 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:26,896 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:04:28,562 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:04:28,563 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:04:28,564 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:04:28,564 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:04:28,861 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:04:30,862 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:04:31,606 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:32,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:04:32,863 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:04:34,864 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:04:36,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:37,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:04:41,784 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:41,878 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:04:46,785 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:47,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:04:48,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:04:48,534 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:04:48,535 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:04:48,869 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:04:49,256 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:04:50,870 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:04:52,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:04:52,871 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:04:53,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:04:53,804 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:04:54,872 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:04:56,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:04:57,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:02,171 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:05:02,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:07,172 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:05:07,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:11,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:05:12,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:13,878 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:05:13,879 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:05:13,880 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:05:13,880 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:05:13,881 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:05:16,882 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:05:17,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:05:18,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:18,883 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:05:22,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:05:23,047 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:23,804 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:05:26,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:05:28,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:32,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:05:32,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:05:32,652 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:05:32,652 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:05:32,652 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:05:32,888 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:05:34,684 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:34,889 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:05:36,890 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:05:38,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:05:39,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:41,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:05:44,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:47,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:05:47,410 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:05:47,411 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:05:47,411 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:05:47,411 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:05:47,895 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:05:48,895 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:05:50,418 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:52,896 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:05:53,102 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:05:53,805 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:05:55,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:05:56,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:06:00,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:02,106 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:06:05,679 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:06:05,680 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:06:05,680 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:06:05,681 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:06:05,901 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:06:06,713 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:06,902 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:06:07,389 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:06:10,903 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:06:11,834 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:11,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:06:16,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:17,401 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:06:21,836 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:22,422 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:06:23,806 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:06:26,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:06:26,961 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:06:26,962 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:06:26,963 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:06:26,963 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:26,964 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:06:27,910 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:06:28,910 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:06:32,211 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:32,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:06:32,912 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:06:37,212 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:37,723 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:06:41,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:06:42,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:46,174 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:06:46,176 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:06:46,176 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:06:46,177 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:06:46,917 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:06:47,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:47,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:06:48,918 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:06:50,919 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:06:52,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:06:52,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:06:53,807 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:06:56,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:06:57,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:01,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:07:02,809 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:06,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:07:07,504 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:07:07,506 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:07:07,506 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:07:07,506 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:07:07,925 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:07:08,538 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:08,925 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:07:11,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:07:12,927 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:07:13,677 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:17,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:07:18,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:22,222 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:07:23,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:23,808 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:07:26,819 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:07:26,821 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:07:26,821 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:07:26,821 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:07:26,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:07:26,932 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:07:28,853 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:28,933 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:07:30,934 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:07:32,550 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:07:33,966 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:37,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:07:38,967 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:41,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:07:43,968 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:47,554 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:07:48,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:49,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:07:49,892 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:07:49,892 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:07:49,892 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:07:49,941 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:07:50,942 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:07:52,671 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:07:53,809 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:07:54,810 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:07:54,943 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:07:56,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:07:59,811 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:02,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:04,812 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:07,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:09,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:10,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:08:10,507 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:08:10,508 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:08:10,508 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:08:10,949 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:08:12,227 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:08:12,950 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:08:14,951 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:08:15,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:17,257 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:20,640 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:22,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:23,810 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:08:25,811 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:26,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:08:26,003 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:08:26,003 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:08:26,004 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:08:26,956 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:08:26,956 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:08:27,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:27,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:27,737 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:08:30,957 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:08:31,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:32,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:36,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:37,742 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:41,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:42,737 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:08:46,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:47,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:49,845 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:08:49,846 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:08:49,846 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:08:49,846 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:08:49,964 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:08:50,964 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:08:51,878 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:08:53,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:08:53,811 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:08:54,966 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:08:57,739 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:08:57,812 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:02,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:03,612 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:09:06,171 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:09:06,173 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:09:06,173 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:09:06,173 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:09:06,970 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:09:08,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:08,889 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:09:08,971 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:09:10,972 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:09:12,738 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:09:13,635 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:17,896 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:09:18,635 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:23,038 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:09:23,638 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:23,816 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:09:27,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:09:27,236 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:09:27,237 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:09:27,237 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:09:27,738 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:09:27,979 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:09:28,979 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:09:29,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:32,802 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:09:32,981 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:09:34,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:37,811 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:09:39,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:42,738 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:09:42,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:09:42,867 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:09:42,868 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:09:42,868 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:09:42,985 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:09:44,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:44,985 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:09:46,986 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:09:48,583 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:09:50,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:53,584 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:09:53,813 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:09:55,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:09:57,738 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:10:00,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:03,586 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:10:05,538 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:10:05,539 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:10:05,540 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:10:05,542 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:10:05,993 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:10:06,571 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:06,994 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:10:09,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:10:10,995 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:10:11,753 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:12,751 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:10:16,753 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:18,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:10:21,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:22,486 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:10:22,487 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:10:22,487 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:10:22,487 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:10:23,000 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:10:23,288 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:10:23,813 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:10:25,000 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:10:27,001 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:10:27,649 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:27,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:10:32,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:33,097 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:10:37,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:38,098 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:10:38,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:10:38,830 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:10:38,830 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:10:38,832 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:10:39,005 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:10:41,006 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:10:42,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:10:43,007 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:10:43,070 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:48,071 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:48,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:10:53,072 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:10:53,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:10:53,815 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:10:57,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:10:58,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:03,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:11:03,523 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:11:03,524 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:11:03,524 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:11:03,647 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:11:04,015 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:11:04,563 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:05,016 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:11:09,017 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:11:09,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:11:09,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:12,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:11:14,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:18,244 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:11:19,661 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:22,843 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:11:22,844 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:11:22,844 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:11:22,846 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:11:23,023 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:11:23,245 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:11:23,816 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:11:24,817 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:25,024 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:11:27,025 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:11:27,740 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:11:30,026 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:33,593 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:11:35,027 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:38,540 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:11:38,542 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:11:38,542 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:11:38,542 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:11:38,594 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:11:39,029 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:11:40,573 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:41,031 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:11:42,741 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:11:43,032 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:11:46,017 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:48,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:11:51,018 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:53,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:11:53,817 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:11:56,819 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:11:57,742 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:11:58,640 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:11:58,642 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:11:58,642 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:11:58,643 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:11:59,037 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:12:01,038 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:12:01,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:03,038 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:12:03,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:12:06,836 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:08,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:12:11,837 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:12,755 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:12:16,838 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:16,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:12:16,901 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:12:16,902 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:12:16,902 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:12:17,051 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:12:18,608 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:12:19,052 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:12:21,053 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:12:22,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:23,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:12:23,818 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:12:27,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:12:27,819 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:32,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:33,616 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:12:37,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:38,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:12:40,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:12:40,737 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:12:40,737 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:12:40,737 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:12:41,060 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:12:43,061 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:12:43,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:12:43,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:45,062 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:12:48,462 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:12:48,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:53,463 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:12:53,820 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:12:54,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:12:58,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:12:59,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:03,470 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:13:04,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:08,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:13:08,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:13:08,513 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:13:08,514 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:13:08,514 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:13:09,071 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:13:10,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:11,072 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:13:13,073 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:13:13,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:13:15,746 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:19,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:13:20,747 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:23,820 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:13:24,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:13:25,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:28,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:13:30,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:33,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:13:33,723 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:13:33,723 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:13:33,723 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:13:34,080 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:13:34,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:13:35,080 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:13:36,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:39,082 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:13:39,444 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:13:41,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:43,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:13:46,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:48,579 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:13:48,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:13:48,903 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:13:48,903 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:13:48,904 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:13:49,086 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:13:51,086 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:13:51,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:53,087 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:13:53,660 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:13:53,821 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:13:57,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:13:58,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:14:02,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:03,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:07,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:08,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:12,113 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:14:12,114 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:14:12,114 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:14:12,115 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:14:13,095 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:14:13,095 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:14:13,146 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:13,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:13,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:14:17,097 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:14:18,294 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:18,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:23,295 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:23,822 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:14:23,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:28,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:14:28,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:30,003 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:14:30,005 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:14:30,005 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:14:30,005 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:14:30,101 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:14:31,102 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:14:34,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:34,625 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:35,103 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:14:39,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:39,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:43,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:14:44,195 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:44,919 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:14:44,920 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:14:44,921 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:14:44,921 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:14:45,107 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:14:47,108 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:14:49,109 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:14:49,637 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:50,134 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:53,823 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:14:54,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:14:55,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:14:58,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:15:00,825 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:03,781 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:15:03,782 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:15:03,782 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:15:03,783 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:15:04,115 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:15:04,655 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:15:05,115 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:15:06,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:09,117 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:15:10,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:15:11,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:13,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:15:16,950 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:18,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:15:18,413 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:15:18,414 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:15:18,414 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:15:19,121 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:15:19,122 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:15:20,174 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:15:22,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:23,123 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:15:23,824 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:15:25,180 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:15:27,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:28,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:15:32,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:34,192 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:15:36,464 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:15:36,466 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:15:36,466 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:15:36,466 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:15:37,128 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:15:37,128 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:15:38,498 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:39,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:15:41,129 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:15:43,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:15:43,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:48,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:49,201 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:15:53,825 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:15:54,201 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:15:54,630 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:15:54,631 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:15:54,631 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:15:54,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:15:54,632 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:15:55,135 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:15:55,135 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:15:58,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:15:59,137 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:15:59,788 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:04,481 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:16:04,789 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:09,488 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:16:09,790 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:11,715 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:16:11,716 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:16:11,717 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:16:11,717 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:16:12,141 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:16:13,142 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:16:14,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:16:14,899 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:17,143 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:16:19,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:16:19,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:23,826 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:16:24,500 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:16:25,827 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:29,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:16:30,828 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:31,565 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:16:31,566 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:16:31,567 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:16:31,567 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:16:32,149 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:16:34,150 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:16:35,286 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:16:35,979 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:36,151 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:16:38,151 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:16:40,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:16:40,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:44,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:16:45,981 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:49,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:16:50,982 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:53,525 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:16:53,527 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:16:53,527 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:16:53,528 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:16:53,827 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:16:54,157 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:16:55,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:16:56,158 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:16:56,595 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:16:58,159 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:16:59,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:17:01,742 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:05,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:17:06,742 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:10,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:17:11,743 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:14,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:17:15,578 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:17:15,579 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:17:15,579 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:17:15,580 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:17:16,166 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:17:17,618 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:18,167 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:17:20,168 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:17:20,289 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:17:22,729 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:23,828 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:17:25,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:17:27,829 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:29,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:17:32,313 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:17:32,314 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:17:32,314 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:17:32,315 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:17:33,173 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:17:33,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:34,173 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:17:35,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:17:38,175 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:17:38,527 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:40,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:17:43,528 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:44,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:17:48,529 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:50,119 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:17:52,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:17:52,559 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:17:52,560 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:17:52,560 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:17:53,181 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:17:53,591 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:53,829 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:17:54,182 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:17:55,268 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:17:58,184 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:17:58,764 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:17:59,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:18:03,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:05,278 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:18:08,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:10,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:18:13,767 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:14,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:18:17,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:18:17,346 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:18:17,346 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:18:17,346 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:18:18,190 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:18:19,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:20,051 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:18:20,191 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:18:22,191 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:18:23,830 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:18:24,830 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:25,249 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:18:29,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:18:29,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:34,832 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:35,266 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:18:36,126 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:18:36,127 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:18:36,127 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:18:36,128 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:18:36,196 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:18:38,197 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:18:40,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:40,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:18:42,198 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:18:44,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:18:45,292 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:49,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:18:50,293 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:53,830 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:18:54,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:18:55,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:18:56,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:18:56,251 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:18:56,251 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:18:56,252 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:18:57,204 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:18:58,205 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:18:59,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:19:01,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:02,206 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:19:05,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:19:06,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:10,049 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:19:11,076 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:19:11,077 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:19:11,077 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:19:11,078 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:19:11,210 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:19:12,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:14,211 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:19:14,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:19:16,212 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:19:17,307 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:19,812 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:19:22,307 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:23,832 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:19:24,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:19:27,833 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:29,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:19:32,834 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:34,815 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:19:36,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:19:36,331 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:19:36,331 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:19:36,331 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:19:37,219 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:19:38,220 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:19:38,364 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:40,052 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:19:42,221 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:19:43,504 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:44,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:19:48,505 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:50,054 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:19:53,505 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:53,833 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:19:54,432 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:19:54,433 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:19:54,433 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:19:54,434 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:19:55,055 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:19:55,226 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:19:56,226 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:19:58,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:19:59,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:20:00,228 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:20:03,586 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:05,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:20:08,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:10,174 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:20:13,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:14,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:20:17,141 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:20:17,142 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:20:17,142 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:20:17,143 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:20:17,233 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:20:19,181 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:19,847 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:20:20,234 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:20:22,235 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:20:23,833 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:20:24,834 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:24,853 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:20:29,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:20:29,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:34,836 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:34,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:20:37,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:20:37,222 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:20:37,222 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:20:37,222 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:20:37,241 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:20:39,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:20:40,242 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:20:40,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:42,243 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:20:44,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:20:45,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:49,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:20:50,412 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:53,834 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:20:54,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:20:55,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:20:55,353 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:20:55,354 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:20:55,354 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:20:56,249 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:20:56,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:20:58,250 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:20:59,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:21:00,250 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:21:01,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:05,304 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:21:06,532 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:10,306 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:21:11,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:14,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:21:16,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:20,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:21:20,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:21:20,369 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:21:20,370 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:21:20,370 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:21:21,258 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:21:22,259 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:21:22,406 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:23,836 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:21:26,092 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:21:26,260 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:21:27,837 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:29,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:21:32,838 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:35,098 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:21:37,311 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:21:37,313 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:21:37,313 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:21:37,313 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:21:38,265 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:21:38,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:40,105 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:21:40,266 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:21:42,266 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:21:43,559 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:44,346 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:21:48,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:50,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:21:53,561 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:53,837 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:21:55,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:21:57,217 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:21:57,218 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:21:57,218 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:21:57,219 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:21:57,272 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:21:59,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:21:59,817 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:22:00,273 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:22:02,274 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:22:04,439 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:04,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:22:09,441 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:09,955 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:22:14,441 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:14,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:22:16,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:22:16,207 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:22:16,207 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:22:16,208 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:22:16,279 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:22:18,280 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:22:19,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:22:20,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:22,281 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:22:23,838 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:22:24,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:22:25,839 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:29,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:22:30,839 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:32,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:22:32,910 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:22:32,910 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:22:32,910 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:22:33,286 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:22:34,286 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:22:35,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:22:35,950 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:38,287 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:22:40,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:22:41,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:44,827 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:22:46,189 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:47,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:22:47,941 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:22:47,942 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:22:48,828 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:22:49,292 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:22:50,292 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:22:51,467 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:22:51,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:53,839 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:22:54,295 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:22:56,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:22:57,840 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:22:59,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:23:02,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:05,730 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:23:07,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:08,635 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:23:08,636 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:23:08,636 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:23:08,637 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:23:09,301 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:23:10,301 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:23:11,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:23:12,844 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:14,303 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:23:14,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:23:17,845 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:20,388 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:23:22,674 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:23:22,675 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:23:22,675 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:23:22,675 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:23:23,307 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:23:23,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:23,839 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:23:24,307 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:23:25,393 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:23:28,308 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:23:28,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:29,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:23:33,836 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:35,596 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:23:37,400 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:23:37,401 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:23:37,402 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:23:37,402 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:23:38,312 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:23:39,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:40,313 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:23:41,141 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:23:42,314 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:23:44,619 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:44,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:23:49,620 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:50,148 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:23:53,840 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:23:54,156 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:23:54,157 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:23:54,157 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:23:54,157 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:23:54,318 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:23:55,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:23:55,874 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:23:56,319 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:23:58,321 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:23:59,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:24:00,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:04,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:24:05,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:08,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:24:08,531 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:24:08,532 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:24:08,534 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:24:09,325 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:24:10,284 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:24:10,326 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:24:10,568 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:14,328 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:24:14,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:24:15,774 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:20,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:24:20,775 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:23,841 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:24:25,302 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:24:25,842 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:29,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:24:30,843 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:31,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:24:31,873 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:24:31,874 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:24:31,874 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:24:32,334 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:24:34,336 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:24:35,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:24:36,065 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:36,336 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:24:40,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:24:41,066 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:44,825 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:24:46,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:50,628 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:24:51,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:53,842 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:24:54,921 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:24:54,922 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:24:54,922 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:24:54,923 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:24:55,343 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:24:55,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:24:56,344 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:24:56,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:24:59,822 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:25:00,345 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:25:02,097 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:05,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:25:07,098 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:10,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:25:12,098 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:14,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:25:15,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:25:15,865 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:25:15,866 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:25:15,866 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:25:16,351 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:25:17,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:18,352 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:25:20,353 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:25:20,469 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:25:23,075 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:23,843 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:25:25,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:25:28,844 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:29,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:25:31,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:25:31,112 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:25:31,112 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:25:31,112 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:25:31,358 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:25:32,358 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:25:34,151 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:34,840 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:25:36,360 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:25:39,300 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:39,851 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:25:44,301 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:44,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:25:48,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:25:48,111 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:25:48,111 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:25:48,111 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:25:48,364 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:25:49,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:25:50,151 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:50,365 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:25:52,366 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:25:53,844 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:25:54,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:25:55,845 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:25:59,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:26:00,845 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:04,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:26:05,846 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:07,031 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:26:07,032 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:26:07,032 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:26:07,033 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:26:07,371 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:26:08,371 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:26:10,653 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:26:11,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:12,373 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:26:14,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:26:16,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:20,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:26:21,222 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:23,845 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:26:25,665 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:26:26,846 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:29,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:26:29,942 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:26:29,944 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:26:29,944 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:26:29,944 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:26:30,380 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:26:31,983 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:32,381 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:26:34,382 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:26:35,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:26:37,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:40,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:26:42,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:44,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:26:46,603 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:26:46,604 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:26:46,604 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:26:46,604 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:26:47,393 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:26:47,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:48,394 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:26:50,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:26:52,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:26:52,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:53,846 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:26:55,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:26:57,847 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:26:59,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:27:02,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:05,337 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:27:06,174 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:27:06,175 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:27:06,176 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:27:06,176 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:27:06,400 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:27:08,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:08,401 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:27:10,402 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:27:10,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:27:13,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:14,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:27:18,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:19,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:27:23,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:23,846 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:27:24,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:27:25,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:27:25,904 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:27:25,904 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:27:25,905 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:27:26,408 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:27:28,409 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:27:28,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:29,824 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:27:30,410 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:27:34,138 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:35,628 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:27:39,139 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:40,634 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:27:42,620 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:27:42,621 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:27:42,622 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:27:42,622 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:27:43,414 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:27:44,415 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:27:44,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:45,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:27:46,415 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:27:48,416 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:27:49,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:50,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:27:53,848 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:27:54,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:27:55,346 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:27:59,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:00,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:28:01,078 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:28:01,079 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:28:01,079 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:28:01,569 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:28:02,422 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:28:02,423 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:28:05,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:05,801 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:28:06,424 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:28:10,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:10,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:28:15,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:28:15,232 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:19,380 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:28:19,381 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:28:19,382 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:28:19,382 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:28:19,429 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:28:20,421 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:20,430 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:28:21,096 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:28:23,848 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:28:24,431 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:28:25,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:26,100 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:28:30,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:28:30,850 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:35,851 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:36,108 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:28:39,285 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:28:39,286 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:28:39,286 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:28:39,287 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:28:39,437 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:28:40,437 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:28:41,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:41,896 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:28:44,439 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:28:45,219 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:28:46,463 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:50,898 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:28:51,464 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:53,849 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:28:55,899 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:28:56,851 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:28:58,248 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:28:58,249 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:28:58,250 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:28:58,250 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:28:58,446 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:29:00,447 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:29:00,883 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:29:02,448 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:29:02,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:05,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:07,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:10,987 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:12,469 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:14,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:29:14,086 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:29:14,086 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:29:14,549 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:29:15,453 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:29:16,272 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:16,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:29:16,454 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:29:17,583 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:18,454 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:29:21,284 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:22,737 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:23,851 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:29:26,293 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:27,852 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:31,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:29:32,853 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:36,481 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:37,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:29:37,368 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:29:37,368 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:29:37,369 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:29:37,462 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:29:38,400 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:38,463 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:29:42,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:42,464 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:29:43,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:46,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:29:48,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:52,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:53,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:29:53,851 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:29:54,341 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:29:54,343 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:29:54,343 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:29:54,343 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:29:54,471 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:29:56,471 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:29:57,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:29:58,472 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:29:59,512 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:01,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:30:04,513 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:07,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:30:09,514 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:12,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:30:14,515 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:16,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:30:16,923 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:30:16,924 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:30:16,924 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:30:16,925 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:30:17,480 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:30:18,480 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:30:19,964 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:21,655 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:30:22,482 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:30:23,852 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:30:25,853 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:26,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:30:30,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:31,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:30:33,171 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:30:33,172 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:30:33,172 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:30:33,173 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:30:33,486 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:30:34,486 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:30:36,212 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:36,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:30:38,488 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:30:41,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:41,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:30:46,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:30:46,632 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:50,150 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:30:50,151 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:30:50,151 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:30:50,151 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:30:50,492 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:30:51,906 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:30:52,191 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:30:52,493 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:30:53,853 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:30:54,494 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:30:56,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:30:57,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:01,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:31:02,855 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:06,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:31:07,856 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:10,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:31:10,495 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:31:10,495 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:31:10,496 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:31:10,499 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:31:12,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:31:12,500 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:31:13,527 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:14,501 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:31:16,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:31:18,681 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:22,256 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:31:23,682 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:23,855 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:31:27,257 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:31:28,552 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:31:28,553 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:31:28,553 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:31:28,553 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:31:29,507 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:31:29,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:30,507 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:31:31,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:31:32,508 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:31:34,761 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:37,306 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:31:39,762 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:42,307 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:31:44,697 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:31:44,698 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:31:44,698 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:31:44,699 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:31:45,512 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:31:45,733 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:46,433 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:31:46,513 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:31:48,514 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:31:50,515 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:31:51,134 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:31:51,439 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:31:53,856 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:31:56,440 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:31:56,857 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:01,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:32:01,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:06,575 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:32:06,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:10,249 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:32:10,251 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:32:10,251 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:32:10,251 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:32:10,522 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:32:11,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:32:12,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:12,523 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:32:14,524 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:32:16,433 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:32:17,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:21,988 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:32:22,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:23,856 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:32:26,994 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:32:27,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:31,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:32:31,991 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:32:31,992 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:32:31,993 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:32:31,993 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:32:32,530 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:32:32,531 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:32:33,032 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:36,531 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:32:36,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:32:38,197 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:41,729 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:32:43,198 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:46,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:32:47,426 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:32:47,427 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:32:47,427 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:32:47,428 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:32:47,535 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:32:48,467 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:48,536 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:32:52,157 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:32:52,537 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:32:53,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:32:53,857 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:32:57,158 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:32:58,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:01,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:33:03,859 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:04,722 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:33:04,723 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:33:04,723 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:33:04,724 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:33:05,542 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:33:06,444 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:33:06,542 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:33:08,543 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:33:08,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:11,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:33:13,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:16,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:33:18,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:21,450 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:33:23,859 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:33:24,859 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:26,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:33:28,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:33:28,288 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:33:28,288 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:33:28,289 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:33:28,549 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:33:30,325 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:30,550 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:33:31,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:33:32,551 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:33:35,562 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:37,076 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:33:40,562 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:42,077 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:33:45,563 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:46,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:33:49,528 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:33:49,529 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:33:49,529 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:33:49,529 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:33:49,556 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:33:50,557 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:33:50,569 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:52,249 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:33:53,859 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:33:54,558 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:33:55,860 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:33:57,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:00,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:01,439 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:34:05,862 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:07,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:08,922 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:34:08,923 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:34:08,924 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:34:08,924 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:34:09,564 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:34:10,564 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:34:10,955 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:12,565 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:34:12,645 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:14,567 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:34:16,341 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:16,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:34:21,341 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:21,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:23,860 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:34:26,655 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:26,861 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:29,561 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:34:29,562 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:34:29,563 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:34:29,563 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:34:29,572 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:34:31,573 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:34:32,187 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:32,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:34:32,595 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:35,575 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:34:37,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:37,760 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:42,324 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:42,761 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:47,187 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:34:47,762 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:52,326 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:52,376 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:34:52,378 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:34:52,378 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:34:52,379 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:34:52,582 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:34:53,409 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:34:53,867 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:34:55,583 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:34:57,584 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:34:58,102 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:34:58,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:02,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:35:03,532 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:08,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:35:08,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:13,118 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:35:13,533 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:17,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:35:18,534 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:19,021 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:35:19,022 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:35:19,022 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:35:19,023 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:35:19,592 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:35:21,593 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:35:22,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:35:23,594 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:35:23,863 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:35:23,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:27,747 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:35:28,864 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:32,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:35:33,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:37,772 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:35:38,579 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:35:38,581 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:35:38,581 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:35:38,583 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:35:38,599 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:35:39,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:41,601 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:35:43,303 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:35:43,602 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:35:44,767 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:47,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:35:49,768 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:52,316 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:35:53,864 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:35:54,865 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:35:57,317 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:35:58,686 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:35:58,687 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:35:58,687 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:35:58,688 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:35:59,607 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:36:00,719 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:01,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:36:02,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:36:03,609 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:36:05,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:07,403 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:36:10,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:12,404 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:36:15,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:17,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:36:18,482 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:36:18,483 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:36:18,483 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:36:18,485 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:36:18,614 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:36:21,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:21,615 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:36:22,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:36:23,616 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:36:23,865 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:36:26,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:27,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:36:31,867 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:32,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:36:36,863 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:36:36,864 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:36:36,864 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:36:36,865 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:36:37,198 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:37,218 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:36:37,621 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:36:39,622 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:36:41,623 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:36:42,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:42,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:36:47,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:36:47,345 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:52,346 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:52,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:36:53,866 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:36:54,966 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:36:54,967 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:36:54,968 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:36:54,968 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:36:55,628 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:36:57,629 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:36:58,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:36:58,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:36:59,630 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:37:02,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:37:03,175 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:07,906 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:37:08,176 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:11,448 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:37:11,450 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:37:11,450 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:37:11,450 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:37:11,635 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:37:13,161 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:37:13,482 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:13,636 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:37:15,637 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:37:17,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:37:17,637 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:37:18,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:23,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:37:23,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:23,867 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:37:28,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:37:28,868 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:32,196 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:37:33,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:34,119 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:37:34,121 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:37:34,121 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:37:34,121 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:37:34,645 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:37:35,645 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:37:37,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:37:39,361 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:39,647 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:37:42,856 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:37:44,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:47,195 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:37:49,362 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:49,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:37:49,959 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:37:49,959 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:37:49,960 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:37:50,652 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:37:51,652 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:37:52,684 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:37:53,868 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:37:54,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:37:55,654 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:37:57,690 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:37:59,869 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:02,195 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:38:04,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:06,685 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:38:06,686 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:38:06,686 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:38:06,687 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:38:07,658 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:38:08,400 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:38:09,659 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:38:10,113 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:11,660 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:38:13,463 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:38:15,269 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:17,207 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:38:20,270 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:21,992 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:38:21,994 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:38:21,994 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:38:21,994 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:38:22,465 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:38:22,664 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:38:23,665 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:38:23,869 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:38:26,157 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:27,685 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:38:27,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:38:31,158 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:32,204 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:38:36,159 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:37,818 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:38:38,589 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:38:38,590 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:38:38,590 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:38:38,591 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:38:38,689 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:38:39,689 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:38:41,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:43,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:38:43,691 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:38:46,811 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:47,204 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:38:51,812 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:52,354 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:38:53,870 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:38:56,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:38:57,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:38:58,156 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:38:58,157 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:38:58,157 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:38:58,157 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:38:58,696 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:38:59,697 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:39:02,204 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:39:02,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:03,698 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:39:07,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:08,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:39:12,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:13,112 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:39:17,136 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:39:17,138 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:39:17,138 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:39:17,138 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:39:17,204 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:39:17,703 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:39:18,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:19,704 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:39:21,705 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:39:22,847 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:39:23,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:23,871 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:39:27,853 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:39:28,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:32,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:39:33,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:36,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:39:36,230 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:39:36,230 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:39:36,230 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:39:36,711 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:39:37,711 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:39:37,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:39:39,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:41,713 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:39:42,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:39:44,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:47,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:39:49,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:51,600 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:39:51,601 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:39:51,602 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:39:51,602 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:39:51,717 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:39:53,336 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:39:53,718 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:39:54,642 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:39:56,283 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:39:57,720 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:39:58,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:00,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:02,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:40:05,558 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:07,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:10,559 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:12,406 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:15,560 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:17,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:40:18,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:40:18,800 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:40:18,800 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:40:18,801 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:40:19,727 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:40:19,727 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:40:20,840 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:22,521 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:23,728 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:40:23,873 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:40:25,874 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:27,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:30,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:32,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:40:35,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:37,529 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:38,295 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:40:38,297 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:40:38,297 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:40:38,299 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:40:38,734 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:40:39,735 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:40:41,329 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:43,036 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:43,737 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:40:46,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:47,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:40:51,477 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:53,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:53,873 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:40:53,978 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:40:53,979 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:40:53,980 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:40:53,980 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:40:54,741 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:40:55,741 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:40:57,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:40:58,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:40:59,743 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:41:02,163 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:02,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:41:07,164 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:07,691 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:41:12,165 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:12,701 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:41:13,538 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:41:13,539 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:41:13,539 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:41:13,541 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:41:13,748 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:41:15,749 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:41:17,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:41:17,733 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:17,749 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:41:22,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:41:22,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:23,874 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:41:27,256 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:41:27,876 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:32,116 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:41:32,117 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:41:32,117 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:41:32,118 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:41:32,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:41:32,755 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:41:33,152 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:33,755 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:41:37,756 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:41:37,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:41:38,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:42,837 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:41:43,453 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:47,207 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:41:48,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:52,844 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:41:53,455 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:53,875 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:41:54,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:41:54,435 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:41:54,435 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:41:54,436 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:41:54,762 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:41:55,763 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:41:58,164 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:41:58,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:41:59,764 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:42:02,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:42:03,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:08,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:42:08,628 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:13,381 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:42:13,628 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:14,325 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:42:14,326 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:42:14,326 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:42:14,327 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:42:14,769 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:42:15,770 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:42:17,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:42:19,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:19,771 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:42:23,035 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:42:23,876 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:42:24,877 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:28,043 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:42:29,878 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:31,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:42:31,881 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:42:31,881 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:42:31,882 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:42:32,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:42:32,776 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:42:33,777 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:42:34,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:35,777 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:42:37,605 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:42:37,778 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:42:40,073 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:42,606 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:42:45,074 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:45,600 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:42:45,601 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:42:45,601 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:42:45,602 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:42:45,781 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:42:47,326 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:42:47,782 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:42:49,783 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:42:50,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:52,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:42:53,878 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:42:55,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:42:57,351 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:42:59,696 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:42:59,697 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:42:59,698 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:42:59,698 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:42:59,786 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:43:01,730 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:01,787 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:43:02,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:43:05,789 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:43:07,231 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:07,474 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:43:12,232 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:12,478 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:43:17,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:17,334 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:43:20,165 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:43:20,167 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:43:20,167 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:43:20,167 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:43:20,795 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:43:21,795 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:43:22,899 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:43:23,204 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:23,879 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:43:25,797 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:43:27,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:43:28,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:32,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:43:33,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:37,530 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:43:37,532 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:43:37,532 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:43:37,533 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:43:37,801 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:43:37,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:43:39,571 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:39,802 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:43:41,803 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:43:43,245 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:43:44,744 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:47,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:43:49,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:53,262 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:43:53,880 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:43:54,177 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:43:54,179 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:43:54,179 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:43:54,179 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:43:54,807 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:43:55,211 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:43:55,808 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:43:58,902 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:43:59,809 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:44:00,356 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:02,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:44:05,357 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:07,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:44:10,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:12,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:44:13,857 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:44:13,858 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:44:13,858 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:44:13,859 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:44:14,815 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:44:15,816 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:44:15,890 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:17,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:44:17,816 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:44:21,013 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:22,579 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:44:23,880 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:44:26,881 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:27,580 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:44:27,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:44:27,728 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:44:27,728 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:44:27,728 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:44:27,820 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:44:29,821 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:44:31,822 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:44:31,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:32,335 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:44:36,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:37,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:44:41,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:42,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:44:46,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:47,337 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:44:50,930 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:44:50,931 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:44:50,931 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:44:50,932 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:44:51,829 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:44:51,829 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:44:51,971 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:52,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:44:53,886 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:44:55,830 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:44:57,156 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:44:57,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:02,157 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:02,336 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:45:07,158 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:07,942 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:12,159 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:12,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:45:12,710 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:45:12,711 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:45:12,711 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:45:12,836 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:45:12,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:13,837 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:45:17,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:45:17,839 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:45:17,879 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:22,444 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:22,880 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:23,882 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:45:27,446 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:27,883 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:32,341 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:45:32,884 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:35,505 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:45:35,507 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:45:35,507 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:45:35,509 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:45:35,846 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:45:37,847 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:45:38,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:38,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:39,848 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:45:43,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:43,656 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:47,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:45:48,657 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:52,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:53,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:45:53,883 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:45:53,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:45:53,955 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:45:53,955 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:45:53,956 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:45:54,855 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:45:55,855 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:45:57,687 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:45:57,856 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:45:59,215 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:02,344 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:46:04,216 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:07,700 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:46:09,217 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:10,426 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:46:10,427 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:46:10,428 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:46:10,430 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:46:10,863 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:46:11,863 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:46:13,155 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:46:14,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:15,865 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:46:17,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:46:19,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:23,169 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:46:23,884 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:46:25,885 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:28,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:46:30,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:32,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:46:35,123 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:46:35,124 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:46:35,124 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:46:35,125 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:46:35,874 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:46:35,874 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:46:36,164 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:37,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:46:39,876 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:46:41,282 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:42,855 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:46:46,283 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:47,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:46:51,284 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:53,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:46:53,685 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:46:53,686 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:46:53,686 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:46:53,688 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:46:53,882 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:46:53,885 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:46:55,883 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:46:56,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:46:57,884 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:46:58,418 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:01,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:02,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:47:06,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:07,425 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:07,964 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:47:07,965 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:47:07,965 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:47:07,966 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:47:08,888 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:47:09,889 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:47:11,890 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:47:12,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:12,711 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:17,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:17,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:47:22,096 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:47:22,097 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:47:22,098 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:47:22,098 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:47:22,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:22,894 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:47:23,137 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:23,886 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:47:23,894 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:47:27,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:27,896 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:47:28,201 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:32,347 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:47:33,202 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:37,843 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:38,203 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:41,493 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:47:41,494 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:47:41,495 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:47:41,495 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:47:41,900 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:47:43,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:43,838 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:43,901 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:47:45,902 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:47:47,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:47:48,968 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:52,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:53,887 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:47:54,888 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:47:57,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:47:59,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:02,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:48:04,893 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:05,408 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:48:05,409 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:48:05,409 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:48:05,410 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:48:05,910 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:48:05,910 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:48:08,142 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:48:09,912 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:48:10,574 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:13,148 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:48:15,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:17,377 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:48:20,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:23,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:48:23,888 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:48:24,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:48:24,239 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:48:24,239 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:48:24,240 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:48:24,917 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:48:25,918 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:48:26,271 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:27,919 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:48:28,962 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:48:31,434 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:32,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:48:36,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:37,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:48:41,436 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:42,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:48:42,253 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:48:42,253 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:48:42,254 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:48:42,925 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:48:42,970 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:48:43,925 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:48:45,926 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:48:46,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:47,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:48:51,477 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:52,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:48:53,889 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:48:56,888 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:48:56,890 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:48:56,890 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:48:56,890 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:48:56,891 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:48:56,930 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:48:57,930 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:48:58,632 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:01,932 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:49:02,115 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:02,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:49:07,116 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:07,647 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:12,116 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:12,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:15,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:49:15,316 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:49:15,317 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:49:15,317 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:49:15,937 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:49:15,938 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:49:17,349 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:17,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:17,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:49:19,939 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:49:22,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:23,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:23,890 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:49:27,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:28,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:32,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:49:32,616 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:49:32,616 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:49:32,617 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:49:32,944 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:49:32,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:49:33,648 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:33,944 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:49:37,946 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:49:38,359 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:38,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:43,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:43,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:47,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:49:48,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:50,538 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:49:50,539 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:49:50,539 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:49:50,540 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:49:50,951 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:49:51,951 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:49:53,270 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:53,891 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:49:53,893 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:49:53,952 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:49:55,952 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:49:58,281 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:49:58,893 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:02,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:50:03,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:07,068 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:50:07,069 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:50:07,069 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:50:07,070 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:50:07,957 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:50:07,957 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:50:08,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:50:09,109 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:11,958 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:50:13,810 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:50:14,245 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:17,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:50:19,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:23,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:50:23,892 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:50:24,893 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:26,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:50:26,946 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:50:26,946 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:50:26,946 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:50:26,964 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:50:27,964 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:50:29,553 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:50:29,979 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:31,966 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:50:32,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:50:35,420 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:38,555 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:50:40,420 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:43,556 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:50:45,421 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:46,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:50:46,856 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:50:46,856 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:50:46,857 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:50:46,971 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:50:47,972 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:50:48,576 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:50:48,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:50:51,044 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:51,974 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:50:53,588 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:50:53,893 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:50:56,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:50:58,607 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:51:01,895 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:03,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:51:04,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:51:04,912 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:51:04,912 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:51:04,912 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:51:04,979 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:51:05,980 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:51:06,951 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:08,642 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:51:09,981 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:51:12,124 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:13,643 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:51:17,125 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:18,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:51:22,126 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:23,654 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:51:23,894 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:51:24,101 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:51:24,102 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:51:24,102 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:51:24,103 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:51:24,987 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:51:25,988 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:51:27,135 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:27,988 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:51:28,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:51:32,273 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:33,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:51:37,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:38,833 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:51:42,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:43,834 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:51:47,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:48,578 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:51:51,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:51:51,231 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:51:51,231 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:51:51,232 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:51:51,997 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:51:51,998 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:51:53,271 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:53,895 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:51:53,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:51:55,999 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:51:58,688 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:51:58,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:52:03,578 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:52:03,688 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:08,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:08,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:52:09,801 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:52:09,802 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:52:09,802 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:52:09,803 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:52:10,004 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:52:10,004 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:52:13,993 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:14,006 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:52:14,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:52:18,578 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:52:18,994 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:23,598 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:52:23,896 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:52:24,897 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:26,727 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:52:26,728 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:52:26,728 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:52:26,729 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:52:27,011 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:52:29,011 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:52:29,353 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:52:29,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:31,012 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:52:33,013 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:52:33,578 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:52:34,935 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:39,355 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:52:39,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:41,427 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:52:41,428 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:52:41,429 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:52:41,429 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:52:42,016 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:52:43,017 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:52:45,139 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:52:45,604 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:47,018 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:52:48,578 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:52:50,605 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:53,897 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:52:54,141 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:52:55,898 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:52:59,155 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:53:00,898 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:01,172 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:53:01,173 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:53:01,174 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:53:01,174 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:53:02,025 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:53:03,025 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:03,772 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:53:06,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:07,027 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:08,894 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:53:11,367 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:13,896 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:53:16,368 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:18,772 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:53:19,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:53:19,359 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:53:19,360 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:53:19,360 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:53:20,032 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:53:21,033 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:21,399 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:23,897 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:53:24,085 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:53:25,034 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:26,898 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:29,094 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:53:31,899 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:33,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:53:36,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:37,504 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:53:37,505 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:53:37,505 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:53:37,506 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:53:38,040 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:53:39,040 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:39,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:53:42,775 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:43,042 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:44,242 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:53:47,776 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:48,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:53:52,777 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:52,858 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:53:52,860 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:53:52,860 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:53:52,860 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:53:53,045 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:53:53,898 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:53:54,598 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:53:55,046 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:57,047 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:58,058 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:53:59,048 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:53:59,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:54:03,059 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:03,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:54:08,060 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:09,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:54:13,060 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:13,592 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:54:13,595 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:54:13,595 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:54:13,595 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:54:14,053 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:54:15,053 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:54:15,297 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:54:18,772 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:54:18,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:19,055 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:54:23,800 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:23,899 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:54:24,307 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:54:28,900 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:29,308 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:54:33,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:54:33,901 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:36,190 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:54:36,191 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:54:36,191 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:54:36,192 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:54:37,062 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:54:38,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:54:39,063 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:54:39,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:41,064 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:54:43,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:54:44,369 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:48,823 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:54:49,370 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:51,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:54:51,009 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:54:51,010 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:54:51,010 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:54:51,067 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:54:53,068 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:54:53,905 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:54:54,726 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:54:55,069 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:54:55,190 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:54:57,070 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:54:59,730 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:55:00,191 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:03,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:55:05,192 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:05,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:55:05,943 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:55:05,943 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:55:05,943 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:55:06,073 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:55:07,074 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:55:09,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:55:10,377 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:11,075 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:55:14,524 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:55:15,378 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:18,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:55:20,379 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:23,901 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:55:24,795 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:55:25,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:55:25,611 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:55:25,611 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:55:25,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:25,613 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:55:26,081 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:55:27,081 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:55:30,341 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:55:30,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:31,083 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:55:33,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:55:35,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:39,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:55:40,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:55:40,393 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:55:40,393 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:55:40,394 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:55:41,086 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:55:41,426 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:43,087 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:55:45,088 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:55:45,124 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:55:46,796 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:48,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:55:51,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:53,902 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:55:54,125 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:55:56,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:55:57,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:55:57,507 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:55:57,507 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:55:57,509 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:55:58,093 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:55:59,094 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:55:59,235 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:02,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:03,095 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:56:03,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:56:07,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:09,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:12,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:14,264 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:17,767 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:18,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:56:22,183 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:56:22,184 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:56:22,184 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:56:22,185 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:56:23,102 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:56:23,103 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:56:23,224 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:23,890 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:23,903 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:56:27,104 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:56:28,389 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:28,903 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:33,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:33,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:56:38,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:38,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:42,289 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:56:42,290 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:56:42,290 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:56:42,292 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:56:43,110 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:56:44,000 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:44,323 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:45,111 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:56:47,112 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:56:48,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:56:49,474 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:53,904 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:56:54,264 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:54,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:56:59,275 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:56:59,905 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:03,057 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:57:03,058 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:57:03,058 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:57:03,058 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:57:03,118 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:57:03,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:57:05,091 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:05,119 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:57:07,120 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:57:09,768 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:57:10,205 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:14,769 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:57:15,206 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:18,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:57:20,207 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:23,905 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:57:24,771 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:57:25,207 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:57:25,208 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:57:25,208 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:57:25,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:25,209 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:57:26,127 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:57:27,128 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:57:29,128 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:57:29,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:57:30,435 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:31,129 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:57:33,809 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:57:35,436 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:38,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:57:40,437 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:42,583 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:57:42,584 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:57:42,584 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:57:42,585 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:57:43,134 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:57:44,290 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:57:45,135 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:57:45,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:47,136 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:57:48,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:57:50,770 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:53,906 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:57:54,309 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:57:55,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:57:59,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:00,483 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:58:00,484 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:58:00,485 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:58:00,485 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:58:01,141 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:58:01,516 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:03,142 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:58:03,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:58:05,143 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:58:06,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:09,203 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:11,651 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:14,211 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:16,652 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:18,040 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:58:18,041 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:58:18,042 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:58:18,042 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:58:18,149 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:58:18,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:58:19,149 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:58:22,272 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:23,151 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:58:23,907 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:58:24,799 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:27,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:29,802 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:32,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:33,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:58:37,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:39,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:42,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:44,793 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:58:44,794 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:58:44,795 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:58:44,795 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:58:44,810 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:45,159 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:58:47,160 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:58:47,963 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:48,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:58:49,161 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:58:52,964 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:53,908 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:58:54,523 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:58,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:58:59,524 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:58:59,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:58:59,766 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:58:59,766 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:58:59,766 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:59:00,165 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:59:01,166 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:59:03,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:59:03,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:05,167 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:59:08,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:09,526 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:59:13,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:14,527 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:59:18,819 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:59:18,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:20,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:59:20,450 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:59:20,451 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:59:20,451 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:59:21,174 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:59:21,174 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:59:23,909 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:59:24,192 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:59:24,910 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:25,175 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:59:29,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:59:29,911 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:33,841 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:59:34,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:39,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:59:39,912 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:43,796 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 08:59:43,797 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 08:59:43,797 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 08:59:43,798 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 08:59:44,183 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 08:59:44,343 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:59:45,183 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:59:45,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:48,818 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 08:59:49,185 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 08:59:50,978 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:54,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 08:59:55,317 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 08:59:56,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 08:59:59,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:00:01,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:01,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:00:01,545 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:00:01,545 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:00:01,546 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:00:02,190 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:00:03,190 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:00:04,159 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:00:06,747 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:07,192 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:00:09,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:00:11,748 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:14,287 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:00:16,749 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:19,158 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:00:19,616 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:00:19,617 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:00:19,617 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:00:19,618 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:00:20,197 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:00:21,197 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:00:22,658 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:23,912 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:00:24,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:00:25,199 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:00:27,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:29,399 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:00:32,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:34,158 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:00:37,764 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:00:37,765 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:00:37,765 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:00:37,767 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:00:38,204 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:00:38,805 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:39,205 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:00:39,488 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:00:43,206 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:00:44,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:44,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:00:49,158 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:00:49,220 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:53,912 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:00:54,513 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:00:54,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:00:57,370 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:00:57,371 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:00:57,371 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:00:57,372 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:00:58,212 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:00:59,212 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:01:00,126 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:01:00,413 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:01,213 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:01:04,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:01:05,613 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:10,139 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:01:10,614 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:15,139 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:01:15,615 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:19,159 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:01:20,616 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:23,913 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:01:25,141 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:01:25,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:26,379 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:01:26,380 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:01:26,380 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:01:26,381 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:01:27,223 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:01:27,223 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:01:31,004 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:01:31,225 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:01:31,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:34,159 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:01:36,601 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:40,008 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:01:41,602 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:45,009 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:01:45,478 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:01:45,479 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:01:45,480 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:01:45,480 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:01:46,231 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:01:47,231 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:01:47,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:49,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:01:49,232 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:01:52,666 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:53,914 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:01:54,478 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:01:57,915 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:01:59,479 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:02:00,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:02:00,641 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:02:00,641 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:02:00,642 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:02:01,237 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:02:01,237 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:02:03,676 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:04,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:02:05,238 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:02:09,061 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:09,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:02:14,061 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:14,251 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:02:19,062 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:19,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:02:19,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:02:19,938 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:02:19,938 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:02:19,939 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:02:20,244 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:02:21,244 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:02:23,915 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:02:24,677 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:02:24,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:25,246 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:02:29,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:02:29,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:34,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:02:34,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:02:34,746 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:02:34,746 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:02:34,747 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:02:35,250 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:02:35,250 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:02:35,786 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:39,251 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:02:39,461 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:02:40,907 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:44,468 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:02:45,908 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:49,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:02:50,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:53,378 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:02:53,379 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:02:53,379 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:02:53,380 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:02:53,916 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:02:54,257 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:02:55,093 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:02:55,258 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:02:55,916 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:02:57,259 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:03:00,100 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:03:01,562 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:04,171 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:03:06,563 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:08,627 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:03:08,628 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:03:08,628 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:03:08,629 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:03:09,263 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:03:09,264 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:03:10,352 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:03:11,669 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:13,265 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:03:15,363 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:03:16,820 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:19,163 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:03:21,821 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:23,917 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:03:24,370 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:03:26,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:29,376 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:03:30,641 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:03:30,642 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:03:30,642 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:03:30,643 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:03:31,272 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:03:31,272 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:03:32,675 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:34,177 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:03:35,273 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:03:37,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:39,383 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:03:42,852 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:44,384 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:03:47,853 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:49,168 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:03:52,854 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:53,918 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:03:54,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:03:55,792 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:03:55,793 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:03:55,793 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:03:55,793 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:03:56,281 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:03:57,281 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:03:58,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:03:59,282 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:04:00,522 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:04:01,282 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:04:04,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:04,168 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:04:09,003 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:09,541 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:04:13,909 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:04:13,911 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:04:13,911 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:04:13,912 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:04:14,287 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:04:14,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:04:14,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:15,288 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:04:19,176 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:04:19,289 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:04:20,136 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:23,918 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:04:24,678 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:04:25,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:29,689 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:04:30,921 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:31,226 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:04:31,227 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:04:31,228 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:04:31,228 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:04:31,294 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:04:33,295 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:04:34,209 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:04:35,296 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:04:36,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:39,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:04:41,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:44,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:04:46,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:48,455 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:04:48,456 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:04:48,457 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:04:48,457 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:04:49,177 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:04:49,301 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:04:49,302 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:04:51,786 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:53,303 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:04:53,924 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:04:54,481 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:04:56,924 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:04:59,482 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:05:01,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:04,200 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:05:06,926 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:08,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:05:08,758 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:05:08,758 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:05:08,759 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:05:09,309 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:05:09,310 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:05:09,484 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:05:11,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:13,311 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:05:14,491 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:05:16,959 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:19,182 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:05:21,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:23,920 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:05:24,786 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:05:25,694 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:05:25,695 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:05:25,696 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:05:25,698 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:05:26,316 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:05:27,316 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:05:27,728 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:29,317 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:05:30,425 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:05:32,891 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:34,183 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:05:37,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:39,427 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:05:42,892 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:44,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:05:44,542 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:05:44,543 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:05:44,543 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:05:44,544 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:05:45,323 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:05:45,323 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:05:48,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:49,183 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:05:49,324 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:05:53,746 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:05:53,921 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:05:55,146 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:05:58,922 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:00,147 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:06:03,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:06:03,640 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:06:03,640 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:06:03,642 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:06:04,183 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:06:04,330 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:06:04,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:05,331 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:06:07,332 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:06:09,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:06:09,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:14,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:06:14,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:19,332 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:06:19,808 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:20,692 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:06:20,694 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:06:20,694 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:06:20,694 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:06:21,337 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:06:21,337 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:06:23,922 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:06:24,405 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:06:24,923 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:25,338 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:06:29,406 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:06:29,924 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:34,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:06:34,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:39,413 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:06:39,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:06:39,545 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:06:39,545 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:06:39,547 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:06:40,343 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:06:40,578 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:41,344 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:06:45,194 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:06:45,345 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:06:45,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:49,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:06:50,758 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:53,923 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:06:55,200 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:06:55,923 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:06:55,965 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:06:55,966 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:06:55,967 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:06:55,967 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:06:56,349 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:06:57,349 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:07:00,683 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:07:01,114 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:01,351 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:07:04,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:07:06,115 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:09,684 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:07:11,116 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:14,571 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:07:14,572 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:07:14,573 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:07:14,575 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:07:14,685 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:07:15,356 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:07:15,356 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:07:16,605 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:19,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:07:19,358 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:07:21,801 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:23,923 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:07:24,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:07:26,925 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:29,475 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:07:31,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:07:31,328 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:07:31,328 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:07:31,328 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:07:31,362 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:07:32,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:33,363 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:07:34,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:07:35,364 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:07:37,509 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:40,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:07:42,510 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:45,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:07:47,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:49,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:07:52,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:53,837 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:07:53,838 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:07:53,838 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:07:53,840 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:07:53,925 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:07:54,371 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:07:55,371 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:07:55,575 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:07:58,047 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:07:59,376 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:08:00,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:08:03,048 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:04,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:08:08,049 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:09,590 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:08:10,716 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:08:10,717 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:08:10,718 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:08:10,718 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:08:11,377 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:08:11,378 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:08:13,750 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:15,337 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:08:15,379 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:08:18,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:19,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:08:23,925 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:08:24,350 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:08:24,926 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:27,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:08:27,240 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:08:27,240 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:08:27,240 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:08:27,384 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:08:29,385 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:08:29,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:08:30,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:31,385 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:08:34,321 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:08:35,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:39,955 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:08:40,671 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:44,956 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:08:45,672 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:49,320 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:08:50,673 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:53,926 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:08:54,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:08:55,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:08:55,259 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:08:55,259 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:08:55,260 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:08:55,394 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:08:56,291 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:08:57,395 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:08:59,396 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:09:00,223 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:01,464 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:04,320 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:09:06,465 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:10,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:11,466 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:13,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:09:13,081 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:09:13,081 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:09:13,081 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:09:13,401 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:09:15,402 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:09:15,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:17,318 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:17,403 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:09:19,341 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:09:22,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:23,927 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:09:24,798 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:27,928 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:29,802 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:30,757 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:09:30,758 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:09:30,759 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:09:30,759 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:09:31,408 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:09:31,409 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:09:33,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:34,326 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:09:35,410 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:09:38,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:39,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:43,914 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:44,474 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:48,832 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:09:48,833 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:09:48,833 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:09:48,834 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:09:49,325 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:09:49,415 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:09:49,415 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:09:49,866 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:53,421 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:09:53,997 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:09:54,586 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:54,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:09:59,587 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:09:59,998 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:04,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:10:04,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:06,443 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:10:06,444 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:10:06,444 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:10:06,445 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:10:07,426 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:10:07,426 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:10:10,162 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:10:10,622 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:11,427 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:10:15,164 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:10:15,623 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:19,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:10:20,624 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:23,004 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:10:23,005 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:10:23,005 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:10:23,431 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:10:23,515 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:10:23,929 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:10:24,733 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:10:25,432 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:10:25,930 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:27,433 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:10:29,745 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:10:31,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:34,327 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:10:36,229 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:39,751 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:10:41,230 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:43,443 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:10:43,444 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:10:43,444 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:10:43,445 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:10:44,441 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:10:45,159 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:10:46,442 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:10:46,476 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:48,443 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:10:49,328 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:10:51,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:10:53,930 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:10:55,182 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:10:56,931 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:00,409 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:11:01,932 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:01,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:11:01,942 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:11:01,942 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:11:01,945 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:11:02,448 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:11:04,449 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:11:04,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:11:06,450 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:11:07,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:08,451 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:11:09,659 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:11:12,108 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:14,670 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:11:17,110 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:19,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:11:21,415 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:11:21,417 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:11:21,417 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:11:21,417 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:11:21,456 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:11:22,456 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:23,931 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:11:24,458 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:11:25,134 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:11:26,459 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:11:27,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:30,136 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:11:32,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:34,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:11:37,193 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:11:37,194 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:11:37,194 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:11:37,196 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:11:37,463 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:11:38,234 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:39,929 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:11:40,464 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:11:42,465 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:11:43,385 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:44,930 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:11:48,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:49,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:11:53,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:53,932 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:11:54,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:11:58,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:11:59,044 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:11:59,045 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:11:59,045 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:11:59,046 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:11:59,472 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:12:00,472 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:12:00,763 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:04,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:04,474 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:12:04,518 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:12:09,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:09,781 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:14,240 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:14,788 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:18,152 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:12:18,153 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:12:18,153 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:12:18,155 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:12:18,480 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:12:19,874 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:19,874 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:19,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:12:20,186 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:20,481 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:12:23,933 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:12:24,486 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:12:24,886 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:25,934 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:29,887 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:30,935 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:34,874 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:12:35,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:38,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:12:38,873 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:12:38,873 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:12:38,873 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:12:39,491 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:12:40,492 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:12:40,595 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:41,913 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:44,493 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:12:45,611 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:47,088 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:49,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:12:52,089 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:53,934 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:12:55,621 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:12:57,935 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:12:58,221 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:12:58,222 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:12:58,223 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:12:58,223 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:12:58,499 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:13:00,499 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:13:00,980 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:13:03,471 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:04,501 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:13:04,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:13:08,472 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:09,986 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:13:13,473 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:14,987 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:13:17,065 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:13:17,066 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:13:17,066 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:13:17,066 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:13:17,506 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:13:18,506 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:13:19,098 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:19,875 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:13:22,508 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:13:23,935 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:13:24,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:25,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:13:29,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:30,904 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:13:31,819 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:13:31,821 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:13:31,821 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:13:32,663 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:13:33,512 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:13:34,512 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:13:35,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:13:35,697 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:38,514 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:13:40,386 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:13:40,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:45,387 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:13:45,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:50,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:13:50,815 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:53,936 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:13:55,324 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:13:55,325 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:13:55,326 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:13:55,326 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:13:55,390 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:13:55,520 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:13:56,365 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:13:58,522 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:14:00,523 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:14:01,094 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:14:01,546 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:05,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:14:06,547 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:11,102 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:14:11,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:16,107 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:14:16,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:19,927 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:14:19,929 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:14:19,929 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:14:19,930 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:14:20,238 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:14:20,531 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:14:21,966 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:22,532 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:14:23,936 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:14:24,533 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:14:25,667 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:14:27,937 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:30,668 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:14:32,938 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:35,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:14:37,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:38,030 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:14:38,032 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:14:38,032 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:14:38,032 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:14:38,539 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:14:40,539 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:14:40,754 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:14:42,540 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:14:43,211 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:45,763 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:14:48,212 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:50,239 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:14:52,852 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:14:52,853 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:14:52,853 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:14:52,854 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:14:53,544 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:14:53,893 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:14:53,937 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:14:54,545 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:14:55,626 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:14:58,546 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:14:59,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:00,639 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:15:04,082 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:05,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:15:09,083 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:10,773 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:15:14,084 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:15,773 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:15:15,972 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:15:15,973 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:15:15,974 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:15:15,974 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:15:16,552 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:15:18,553 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:15:19,188 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:20,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:15:20,554 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:15:23,938 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:15:24,939 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:25,581 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:15:29,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:30,582 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:15:34,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:35,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:15:35,461 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:15:35,462 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:15:35,462 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:15:35,463 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:15:35,559 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:15:38,560 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:15:40,561 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:15:40,629 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:41,170 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:15:45,630 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:46,171 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:15:50,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:15:50,631 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:53,939 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:15:55,940 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:15:56,172 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:15:58,497 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:15:58,498 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:15:58,498 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:15:58,498 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:15:58,568 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:16:00,569 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:16:01,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:16:01,531 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:04,570 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:16:05,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:16:06,766 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:10,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:16:11,767 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:15,255 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:16:16,768 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:17,366 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:16:17,367 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:16:17,368 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:16:17,368 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:16:17,575 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:16:18,575 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:16:20,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:16:22,564 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:22,577 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:16:23,940 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:16:26,079 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:16:27,941 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:31,081 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:16:32,942 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:35,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:16:37,063 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:16:37,064 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:16:37,065 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:16:37,065 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:16:37,582 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:16:38,104 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:38,582 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:16:40,777 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:16:42,584 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:16:43,212 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:45,778 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:16:48,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:50,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:16:53,213 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:16:53,941 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:16:55,831 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:16:58,942 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:00,835 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:17:02,236 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:17:02,237 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:17:02,238 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:17:02,238 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:17:02,592 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:17:04,276 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:04,593 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:17:05,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:17:06,594 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:17:09,442 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:10,976 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:17:14,443 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:15,978 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:17:19,444 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:20,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:17:20,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:17:20,343 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:17:20,344 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:17:20,344 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:17:20,599 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:17:22,600 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:17:23,942 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:17:24,601 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:17:24,943 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:26,074 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:17:29,944 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:31,084 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:17:34,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:35,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:17:39,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:41,086 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:17:42,217 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:17:42,218 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:17:42,218 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:17:42,218 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:17:42,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:17:44,608 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:17:45,250 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:46,609 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:17:46,794 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:17:50,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:17:50,371 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:53,943 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:17:55,807 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:17:55,944 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:17:59,781 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:17:59,782 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:17:59,782 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:17:59,783 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:18:00,615 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:18:01,506 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:18:01,815 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:02,616 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:18:04,617 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:18:05,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:18:07,006 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:10,513 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:18:12,007 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:15,409 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:18:15,411 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:18:15,411 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:18:15,411 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:18:15,517 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:18:15,621 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:18:16,621 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:18:17,444 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:20,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:18:20,623 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:18:22,597 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:23,943 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:18:26,160 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:18:27,945 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:30,305 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:18:30,306 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:18:30,306 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:18:30,307 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:18:30,626 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:18:31,167 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:18:32,627 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:18:33,340 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:34,628 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:18:35,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:18:38,510 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:40,917 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:18:43,511 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:45,918 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:18:46,849 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:18:46,850 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:18:46,850 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:18:46,850 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:18:47,632 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:18:48,633 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:18:48,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:50,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:18:52,635 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:18:53,944 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:18:53,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:18:55,571 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:18:58,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:00,577 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:19:01,564 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:19:01,565 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:19:01,566 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:19:01,566 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:19:01,640 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:19:02,640 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:19:04,604 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:05,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:19:06,642 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:19:09,764 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:10,330 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:19:14,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:15,331 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:19:16,248 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:19:16,249 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:19:16,249 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:19:16,250 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:19:16,645 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:19:18,646 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:19:20,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:19:20,438 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:20,647 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:19:23,945 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:19:25,946 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:25,971 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:19:30,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:30,972 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:19:35,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:19:35,948 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:39,241 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:19:39,242 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:19:39,242 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:19:39,243 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:19:39,653 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:19:40,654 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:19:40,977 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:19:41,282 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:44,655 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:19:46,001 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:19:46,444 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:50,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:19:51,445 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:19:53,946 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:19:56,002 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:19:56,947 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:01,004 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:01,783 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:20:01,784 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:20:01,784 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:20:01,785 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:20:02,662 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:20:02,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:04,663 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:20:05,253 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:20:06,664 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:20:07,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:10,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:12,921 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:15,500 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:17,922 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:20,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:20:22,923 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:23,947 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:20:25,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:27,536 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:20:27,537 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:20:27,537 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:20:27,538 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:20:27,671 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:20:28,570 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:28,671 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:20:31,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:32,673 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:20:33,728 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:35,257 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:20:38,729 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:40,260 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:43,730 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:45,261 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:48,371 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:20:48,372 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:20:48,373 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:20:48,373 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:20:48,678 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:20:49,404 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:50,679 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:20:50,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:50,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:20:52,680 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:20:53,948 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:20:54,949 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:20:56,103 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:20:59,950 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:01,201 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:04,503 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:21:04,504 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:21:04,504 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:21:04,504 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:21:04,684 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:21:05,537 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:06,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:06,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:06,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:21:06,685 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:21:08,686 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:21:10,734 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:11,252 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:15,735 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:16,254 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:20,736 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:21,029 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:21:21,030 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:21:21,030 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:21:21,031 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:21:21,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:21:21,690 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:21:22,691 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:21:23,949 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:21:26,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:26,692 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:21:26,768 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:31,453 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:31,789 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:36,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:21:36,454 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:41,455 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:41,790 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:46,154 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:21:46,155 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:21:46,155 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:21:46,156 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:21:46,700 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:21:46,791 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:47,190 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:48,701 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:21:50,702 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:21:51,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:21:52,433 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:21:53,950 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:21:56,920 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:21:57,951 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:01,921 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:22:02,952 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:06,247 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:22:06,299 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:22:06,300 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:22:06,300 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:22:06,300 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:22:06,707 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:22:08,339 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:08,709 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:22:10,710 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:22:12,063 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:22:13,662 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:17,076 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:22:18,663 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:21,249 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:22:23,664 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:23,951 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:22:25,755 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:22:25,756 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:22:25,757 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:22:25,757 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:22:26,715 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:22:26,716 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:22:27,487 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:22:28,797 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:30,717 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:22:32,494 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:22:33,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:36,248 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:22:38,936 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:41,501 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:22:43,848 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:22:43,849 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:22:43,849 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:22:43,850 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:22:44,722 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:22:44,723 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:22:44,882 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:46,585 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:22:48,724 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:22:50,067 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:51,262 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:22:53,952 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:22:55,953 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:22:56,593 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:00,954 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:01,594 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:03,430 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:23:03,432 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:23:03,432 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:23:03,432 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:23:03,730 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:23:04,730 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:23:06,319 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:23:06,469 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:08,732 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:23:12,014 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:12,255 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:17,015 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:17,256 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:21,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:23:22,016 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:23,279 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:23:23,280 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:23:23,281 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:23:23,281 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:23:23,738 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:23:23,954 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:23:24,738 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:23:27,044 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:27,548 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:28,740 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:23:32,045 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:32,549 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:36,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:23:37,550 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:42,057 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:42,551 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:43,047 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:23:43,048 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:23:43,048 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:23:43,049 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:23:43,746 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:23:44,746 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:23:47,806 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:48,321 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:48,748 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:23:51,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:23:53,322 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:23:53,955 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:23:56,818 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:23:58,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:00,878 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:24:00,880 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:24:00,880 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:24:00,880 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:24:01,754 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:24:02,617 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:24:02,754 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:24:04,125 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:04,755 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:24:06,262 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:24:09,126 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:11,642 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:24:14,127 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:16,644 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:24:17,779 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:24:17,780 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:24:17,780 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:24:17,781 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:24:18,760 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:24:18,761 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:24:19,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:21,259 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:24:22,762 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:24:23,956 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:24:24,956 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:26,428 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:24:29,957 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:31,437 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:24:34,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:36,259 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:24:36,342 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:24:36,343 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:24:36,343 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:24:36,344 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:24:36,767 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:24:38,768 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:24:40,763 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:40,769 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:24:42,062 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:24:45,764 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:47,068 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:24:50,764 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:51,259 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:24:53,957 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:24:55,958 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:24:57,075 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:00,959 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:02,080 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:04,360 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:25:04,362 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:25:04,362 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:25:04,364 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:25:04,777 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:25:06,397 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:06,778 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:25:07,020 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:25:08,782 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:25:11,609 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:12,097 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:16,610 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:17,104 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:19,650 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:25:19,651 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:25:19,652 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:25:19,910 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:25:20,787 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:25:20,787 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:25:21,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:22,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:22,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:25:23,958 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:25:24,788 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:25:27,680 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:27,959 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:32,683 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:32,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:37,543 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:25:37,961 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:41,233 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:25:41,235 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:25:41,235 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:25:41,237 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:25:41,795 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:25:42,795 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:25:42,950 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:43,274 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:44,796 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:25:46,797 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:25:47,999 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:48,391 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:52,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:25:53,392 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:25:53,959 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:25:58,015 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:25:58,146 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:25:58,147 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:25:58,147 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:25:58,148 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:25:58,802 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:25:58,803 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:25:59,180 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:02,803 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:03,763 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:26:04,357 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:07,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:26:09,357 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:12,765 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:26:14,358 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:15,315 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:26:15,316 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:26:15,316 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:26:15,318 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:26:15,808 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:26:16,808 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:18,029 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:26:18,809 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:19,499 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:20,810 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:22,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:26:23,959 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:26:24,960 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:28,037 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:26:29,961 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:33,046 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:26:34,962 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:37,222 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:26:37,223 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:26:37,223 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:26:37,223 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:26:37,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:26:37,817 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:26:38,817 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:40,257 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:40,818 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:42,818 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:42,826 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:26:45,451 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:47,996 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:26:50,452 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:52,196 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:26:52,198 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:26:52,198 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:26:52,200 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:26:52,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:26:52,822 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:26:52,822 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:53,961 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:26:55,706 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:26:56,824 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:26:57,838 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:00,707 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:02,850 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:05,708 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:07,545 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:27:10,709 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:12,852 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:13,416 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:27:13,417 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:27:13,417 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:27:13,418 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:27:13,831 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:27:14,831 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:27:16,449 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:18,246 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:18,833 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:27:21,616 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:22,544 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:27:23,961 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:27:26,962 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:28,258 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:31,687 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:27:31,688 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:27:31,688 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:27:31,688 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:27:31,837 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:27:32,720 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:32,838 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:27:33,397 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:34,732 DEBUG SenderThread:1091 [sender.py:send():379] send: telemetry +2024-04-11 09:27:34,733 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:27:34,735 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: partial_history +2024-04-11 09:27:34,736 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:27:34,736 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:27:34,737 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:27:34,737 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:27:34,737 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:27:34,737 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:27:34,738 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:27:34,738 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:27:34,738 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:27:34,739 DEBUG SenderThread:1091 [sender.py:send():379] send: history +2024-04-11 09:27:34,739 DEBUG SenderThread:1091 [sender.py:send_request():406] send_request: summary_record +2024-04-11 09:27:34,739 INFO SenderThread:1091 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-11 09:27:34,838 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/wandb-summary.json +2024-04-11 09:27:36,839 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/output.log +2024-04-11 09:27:37,870 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:39,812 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:39,812 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:39,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:27:42,871 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:45,814 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:47,872 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:50,815 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:27:52,873 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:53,962 DEBUG SenderThread:1091 [sender.py:send():379] send: stats +2024-04-11 09:27:54,813 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-11 09:27:57,963 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:27:59,816 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive +2024-04-11 09:28:02,969 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: status_report +2024-04-11 09:28:03,848 INFO Thread-12 :1091 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240411_040722-rimyh9a1/files/config.yaml +2024-04-11 09:28:04,817 DEBUG HandlerThread:1091 [handler.py:handle_request():146] handle_request: keepalive