diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" --- "a/wandb/debug-internal.log" +++ "b/wandb/debug-internal.log" @@ -1,32 +1,32 @@ -2024-04-12 11:59:51,590 INFO StreamThr :151 [internal.py:wandb_internal():86] W&B internal server running at pid: 151, started at: 2024-04-12 11:59:51.589978 -2024-04-12 11:59:51,592 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status -2024-04-12 11:59:52,113 INFO WriterThread:151 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/run-o0z5y86z.wandb -2024-04-12 11:59:52,113 DEBUG SenderThread:151 [sender.py:send():379] send: header -2024-04-12 11:59:52,116 DEBUG SenderThread:151 [sender.py:send():379] send: run -2024-04-12 11:59:52,257 INFO SenderThread:151 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files -2024-04-12 11:59:52,257 INFO SenderThread:151 [sender.py:_start_run_threads():1124] run started: o0z5y86z with start time 1712923191.59097 -2024-04-12 11:59:52,265 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: check_version -2024-04-12 11:59:52,265 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: check_version -2024-04-12 11:59:52,355 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: run_start -2024-04-12 11:59:52,366 DEBUG HandlerThread:151 [system_info.py:__init__():26] System info init -2024-04-12 11:59:52,366 DEBUG HandlerThread:151 [system_info.py:__init__():41] System info init done -2024-04-12 11:59:52,366 INFO HandlerThread:151 [system_monitor.py:start():194] Starting system monitor -2024-04-12 11:59:52,366 INFO SystemMonitor:151 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-04-12 11:59:52,366 INFO HandlerThread:151 [system_monitor.py:probe():214] Collecting system info -2024-04-12 11:59:52,367 INFO SystemMonitor:151 [interfaces.py:start():190] Started cpu monitoring -2024-04-12 11:59:52,368 INFO SystemMonitor:151 [interfaces.py:start():190] Started disk monitoring -2024-04-12 11:59:52,369 INFO SystemMonitor:151 [interfaces.py:start():190] Started gpu monitoring -2024-04-12 11:59:52,370 INFO SystemMonitor:151 [interfaces.py:start():190] Started memory monitoring -2024-04-12 11:59:52,371 INFO SystemMonitor:151 [interfaces.py:start():190] Started network monitoring -2024-04-12 11:59:52,385 DEBUG HandlerThread:151 [system_info.py:probe():150] Probing system -2024-04-12 11:59:52,387 DEBUG HandlerThread:151 [gitlib.py:_init_repo():56] git repository is invalid -2024-04-12 11:59:52,388 DEBUG HandlerThread:151 [system_info.py:probe():198] Probing system done -2024-04-12 11:59:52,388 DEBUG HandlerThread:151 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-04-12T11:59:52.385897', 'startedAt': '2024-04-12T11:59:51.583216', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': 'f044666cc877', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.152, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.152, 'min': 0.0, 'max': 0.0}, {'current': 2000.152, 'min': 0.0, 'max': 0.0}, {'current': 2000.152, 'min': 0.0, 'max': 0.0}, {'current': 2000.152, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5565.786502838135}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.357559204101562}} -2024-04-12 11:59:52,388 INFO HandlerThread:151 [system_monitor.py:probe():224] Finished collecting system info -2024-04-12 11:59:52,388 INFO HandlerThread:151 [system_monitor.py:probe():227] Publishing system info -2024-04-12 11:59:52,388 DEBUG HandlerThread:151 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment -2024-04-12 11:59:53,259 INFO Thread-12 :151 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/conda-environment.yaml -2024-04-12 12:00:07,403 ERROR HandlerThread:151 [system_info.py:_save_conda():221] Error saving conda packages: Command '['conda', 'env', 'export']' timed out after 15 seconds +2024-04-13 02:10:55,004 INFO StreamThr :145 [internal.py:wandb_internal():86] W&B internal server running at pid: 145, started at: 2024-04-13 02:10:55.003701 +2024-04-13 02:10:55,006 DEBUG HandlerThread:145 [handler.py:handle_request():146] handle_request: status +2024-04-13 02:10:55,420 INFO WriterThread:145 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240413_021054-hjdgsc19/run-hjdgsc19.wandb +2024-04-13 02:10:55,420 DEBUG SenderThread:145 [sender.py:send():379] send: header +2024-04-13 02:10:55,424 DEBUG SenderThread:145 [sender.py:send():379] send: run +2024-04-13 02:10:55,567 INFO SenderThread:145 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240413_021054-hjdgsc19/files +2024-04-13 02:10:55,567 INFO SenderThread:145 [sender.py:_start_run_threads():1124] run started: hjdgsc19 with start time 1712974255.004684 +2024-04-13 02:10:55,575 DEBUG HandlerThread:145 [handler.py:handle_request():146] handle_request: check_version +2024-04-13 02:10:55,575 DEBUG SenderThread:145 [sender.py:send_request():406] send_request: check_version +2024-04-13 02:10:55,667 DEBUG HandlerThread:145 [handler.py:handle_request():146] handle_request: run_start +2024-04-13 02:10:55,678 DEBUG HandlerThread:145 [system_info.py:__init__():26] System info init +2024-04-13 02:10:55,678 DEBUG HandlerThread:145 [system_info.py:__init__():41] System info init done +2024-04-13 02:10:55,678 INFO HandlerThread:145 [system_monitor.py:start():194] Starting system monitor +2024-04-13 02:10:55,679 INFO SystemMonitor:145 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-13 02:10:55,679 INFO SystemMonitor:145 [interfaces.py:start():190] Started cpu monitoring +2024-04-13 02:10:55,680 INFO HandlerThread:145 [system_monitor.py:probe():214] Collecting system info +2024-04-13 02:10:55,680 INFO SystemMonitor:145 [interfaces.py:start():190] Started disk monitoring +2024-04-13 02:10:55,681 INFO SystemMonitor:145 [interfaces.py:start():190] Started gpu monitoring +2024-04-13 02:10:55,682 INFO SystemMonitor:145 [interfaces.py:start():190] Started memory monitoring +2024-04-13 02:10:55,682 INFO SystemMonitor:145 [interfaces.py:start():190] Started network monitoring +2024-04-13 02:10:55,693 DEBUG HandlerThread:145 [system_info.py:probe():150] Probing system +2024-04-13 02:10:55,695 DEBUG HandlerThread:145 [gitlib.py:_init_repo():56] git repository is invalid +2024-04-13 02:10:55,695 DEBUG HandlerThread:145 [system_info.py:probe():198] Probing system done +2024-04-13 02:10:55,695 DEBUG HandlerThread:145 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-04-13T02:10:55.693406', 'startedAt': '2024-04-13T02:10:54.997565', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': 'a7e681bf603a', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.182, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.182, 'min': 0.0, 'max': 0.0}, {'current': 2000.182, 'min': 0.0, 'max': 0.0}, {'current': 2000.182, 'min': 0.0, 'max': 0.0}, {'current': 2000.182, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5569.521137237549}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.357559204101562}} +2024-04-13 02:10:55,695 INFO HandlerThread:145 [system_monitor.py:probe():224] Finished collecting system info +2024-04-13 02:10:55,695 INFO HandlerThread:145 [system_monitor.py:probe():227] Publishing system info +2024-04-13 02:10:55,695 DEBUG HandlerThread:145 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-04-13 02:10:56,569 INFO Thread-12 :145 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240413_021054-hjdgsc19/files/conda-environment.yaml +2024-04-13 02:11:10,710 ERROR HandlerThread:145 [system_info.py:_save_conda():221] Error saving conda packages: Command '['conda', 'env', 'export']' timed out after 15 seconds Traceback (most recent call last): File "/opt/conda/lib/python3.10/site-packages/wandb/sdk/internal/system/system_info.py", line 214, in _save_conda subprocess.call( @@ -37,13934 +37,63 @@ Traceback (most recent call last): File "/opt/conda/lib/python3.10/subprocess.py", line 1951, in _wait raise TimeoutExpired(self.args, timeout) subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after 15 seconds -2024-04-12 12:00:07,406 DEBUG HandlerThread:151 [system_info.py:_save_conda():222] Saving conda packages done -2024-04-12 12:00:07,406 INFO HandlerThread:151 [system_monitor.py:probe():229] Finished publishing system info -2024-04-12 12:00:07,413 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:00:07,413 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: keepalive -2024-04-12 12:00:07,414 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:00:07,414 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: keepalive -2024-04-12 12:00:07,414 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:00:07,414 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: keepalive -2024-04-12 12:00:07,415 DEBUG SenderThread:151 [sender.py:send():379] send: files -2024-04-12 12:00:07,415 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-metadata.json with policy now -2024-04-12 12:00:07,627 INFO wandb-upload_0:151 [upload_job.py:push():131] Uploaded file /tmp/tmp8a4k92xgwandb/a6r39mna-wandb-metadata.json -2024-04-12 12:00:08,262 INFO Thread-12 :151 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-metadata.json -2024-04-12 12:00:08,390 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: python_packages -2024-04-12 12:00:08,391 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: python_packages -2024-04-12 12:00:08,394 DEBUG SenderThread:151 [sender.py:send():379] send: telemetry -2024-04-12 12:00:08,404 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:00:08,406 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:00:08,406 DEBUG SenderThread:151 [sender.py:send():379] send: config -2024-04-12 12:00:08,408 DEBUG SenderThread:151 [sender.py:send():379] send: metric -2024-04-12 12:00:08,408 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:00:08,482 DEBUG SenderThread:151 [sender.py:send():379] send: telemetry -2024-04-12 12:00:08,482 DEBUG SenderThread:151 [sender.py:send():379] send: metric -2024-04-12 12:00:08,482 WARNING SenderThread:151 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) -2024-04-12 12:00:08,482 DEBUG SenderThread:151 [sender.py:send():379] send: telemetry -2024-04-12 12:00:09,262 INFO Thread-12 :151 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/requirements.txt -2024-04-12 12:00:10,263 INFO Thread-12 :151 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:00:12,264 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:00:13,323 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:00:18,324 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:00:23,329 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:00:23,403 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:00:23,404 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:00:23,439 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:00:24,268 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/config.yaml -2024-04-12 12:00:27,602 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:00:27,605 DEBUG SenderThread:151 [sender.py:send():379] send: metric -2024-04-12 12:00:27,605 DEBUG SenderThread:151 [sender.py:send():379] send: metric -2024-04-12 12:00:27,605 DEBUG SenderThread:151 [sender.py:send():379] send: metric -2024-04-12 12:00:27,605 DEBUG SenderThread:151 [sender.py:send():379] send: metric -2024-04-12 12:00:27,605 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:00:27,606 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:00:27,608 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:00:28,270 INFO Thread-12 :151 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:00:29,389 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:00:30,271 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:00:34,390 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:00:38,391 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:00:38,392 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:00:38,392 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:00:39,502 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:00:44,503 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:00:47,245 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:00:47,246 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:00:47,247 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:00:47,249 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:00:47,278 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:00:49,969 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:00:50,279 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:00:52,372 DEBUG SystemMonitor:151 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-04-12 12:00:52,373 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:00:53,393 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:00:53,394 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:00:53,397 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:00:55,467 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:00:56,281 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/config.yaml -2024-04-12 12:01:00,558 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:01:05,385 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:01:05,387 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:01:05,387 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:01:05,387 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:01:06,097 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:01:06,286 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:01:08,287 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:01:08,391 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:01:08,392 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:01:08,395 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:01:11,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:01:16,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:01:21,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:01:22,374 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:01:23,392 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:01:23,392 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:01:23,433 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:01:26,153 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:01:26,154 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:01:26,154 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:01:26,155 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:01:26,295 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:01:26,895 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:01:28,296 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:01:31,896 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:01:36,897 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:01:38,392 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:01:38,393 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:01:38,393 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:01:42,504 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:01:45,325 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:01:45,327 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:01:45,327 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:01:45,329 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:01:46,304 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:01:48,048 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:01:48,305 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:01:52,375 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:01:53,376 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:01:53,392 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:01:53,393 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:01:53,393 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:01:58,507 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:02:03,508 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:02:05,820 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:02:05,822 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:02:05,822 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:02:05,822 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:02:06,311 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:02:08,312 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:02:08,392 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:02:08,393 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:02:08,395 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:02:08,520 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:02:13,521 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:02:18,522 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:02:22,376 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:02:23,392 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:02:23,393 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:02:23,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:02:24,432 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:02:26,475 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:02:26,476 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:02:26,476 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:02:26,477 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:02:27,319 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:02:30,282 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:02:30,320 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:02:35,283 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:02:38,393 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:02:38,393 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:02:38,396 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:02:40,499 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:02:45,500 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:02:46,048 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:02:46,049 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:02:46,049 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:02:46,052 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:02:46,326 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:02:48,328 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:02:50,504 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:02:52,377 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:02:53,393 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:02:53,393 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:02:53,396 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:02:55,517 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:03:00,518 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:03:05,519 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:03:08,073 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:03:08,074 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:03:08,075 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:03:08,075 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:03:08,337 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:03:08,527 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:03:08,528 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:03:08,529 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:03:10,338 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:03:10,657 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:03:15,658 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:03:20,659 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:03:22,378 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:03:23,493 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:03:23,494 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:03:23,499 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:03:26,585 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:03:28,472 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:03:28,474 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:03:28,474 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:03:28,474 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:03:29,344 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:03:30,345 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:03:31,916 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:03:36,917 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:03:38,493 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:03:38,494 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:03:38,499 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:03:42,611 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:03:47,612 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:03:49,063 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:03:49,064 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:03:49,064 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:03:49,066 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:03:49,352 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:03:52,353 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:03:52,378 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:03:53,380 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:03:53,493 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:03:53,494 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:03:53,499 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:03:58,602 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:04:03,603 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:04:08,493 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:04:08,494 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:04:08,533 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:04:08,609 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:04:10,024 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:04:10,026 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:04:10,026 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:04:10,026 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:04:10,360 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:04:12,361 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:04:13,823 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:04:18,824 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:04:22,379 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:04:23,494 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:04:23,494 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:04:23,499 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:04:24,609 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:04:29,610 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:04:32,124 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:04:32,125 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:04:32,125 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:04:32,127 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:04:32,368 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:04:34,369 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:04:34,886 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:04:38,494 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:04:38,494 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:04:38,499 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:04:40,606 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:04:45,607 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:04:50,271 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:04:50,272 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:04:50,273 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:04:50,273 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:04:50,375 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:04:51,119 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:04:52,376 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:04:52,381 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:04:53,494 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:04:53,494 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:04:53,499 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:04:56,630 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:05:01,631 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:05:06,632 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:05:08,494 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:05:08,495 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:05:08,534 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:05:10,881 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:05:10,882 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:05:10,882 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:05:10,883 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:05:11,383 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:05:12,624 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:05:14,384 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:05:17,625 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:05:22,382 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:05:23,383 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:05:23,494 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:05:23,495 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:05:23,499 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:05:28,622 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:05:32,142 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:05:32,143 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:05:32,144 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:05:32,144 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:05:32,392 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:05:33,886 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:05:34,393 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:05:38,494 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:05:38,495 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:05:38,499 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:05:39,603 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:05:44,604 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:05:49,606 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:05:52,383 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:05:53,276 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:05:53,278 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:05:53,278 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:05:53,278 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:05:53,399 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:05:53,753 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:05:53,753 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:05:53,754 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:05:55,039 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:05:56,401 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:06:00,040 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:06:05,041 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:06:08,719 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:06:08,720 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:06:08,728 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:06:10,842 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:06:12,886 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:06:12,887 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:06:12,887 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:06:12,888 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:06:13,406 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:06:16,408 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:06:16,621 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:06:21,622 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:06:22,384 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:06:23,719 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:06:23,720 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:06:23,728 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:06:26,806 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:06:31,807 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:06:34,166 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:06:34,168 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:06:34,168 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:06:34,168 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:06:34,414 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:06:36,415 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:06:36,986 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:06:38,719 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:06:38,719 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:06:38,728 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:06:42,852 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:06:47,853 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:06:52,385 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:06:53,385 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:06:53,719 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:06:53,719 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:06:53,759 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:06:56,572 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:06:56,574 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:06:56,574 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:06:56,575 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:06:57,423 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:06:58,423 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:06:59,337 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:07:04,338 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:07:08,719 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:07:08,720 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:07:08,729 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:07:09,763 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:07:14,764 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:07:16,657 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:07:16,658 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:07:16,658 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:07:16,659 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:07:17,431 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:07:20,392 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:07:20,432 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:07:22,386 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:07:23,719 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:07:23,720 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:07:23,729 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:07:25,851 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:07:30,852 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:07:35,673 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:07:35,674 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:07:35,674 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:07:35,675 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:07:36,439 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:07:36,675 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:07:38,440 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:07:38,719 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:07:38,720 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:07:38,748 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:07:41,858 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:07:46,859 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:07:51,860 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:07:52,387 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:07:53,720 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:07:53,720 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:07:53,760 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:07:56,960 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:07:56,961 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:07:56,961 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:07:56,962 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:07:56,962 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:07:57,447 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:08:00,448 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:08:02,699 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:08:07,700 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:08:08,720 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:08:08,720 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:08:08,729 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:08:12,822 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:08:17,672 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:08:17,673 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:08:17,673 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:08:17,674 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:08:18,414 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:08:18,455 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:08:20,456 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:08:22,388 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:08:23,720 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:08:23,720 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:08:23,721 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:08:23,729 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:08:28,754 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:08:33,754 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:08:38,720 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:08:38,721 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:08:38,761 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:08:38,855 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:08:40,016 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:08:40,017 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:08:40,018 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:08:40,018 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:08:40,465 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:08:42,465 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:08:44,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:08:49,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:08:52,389 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:08:53,720 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:08:53,721 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:08:53,730 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:08:54,774 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:08:59,776 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:09:02,614 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:09:02,616 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:09:02,616 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:09:02,616 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:09:03,473 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:09:04,474 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:09:05,405 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:09:08,721 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:09:08,721 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:09:08,730 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:09:10,810 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:09:15,811 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:09:20,812 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:09:22,389 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:09:23,721 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:09:23,721 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:09:23,747 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:09:25,207 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:09:25,208 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:09:25,208 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:09:25,209 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:09:25,482 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:09:25,949 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:09:28,483 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:09:30,950 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:09:35,951 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:09:38,720 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:09:38,721 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:09:38,730 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:09:41,844 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:09:45,515 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:09:45,517 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:09:45,517 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:09:45,517 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:09:46,490 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:09:47,231 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:09:48,491 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:09:52,232 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:09:52,391 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:09:53,721 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:09:53,722 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:09:53,731 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:09:57,856 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:10:02,857 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:10:07,657 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:10:07,658 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:10:07,658 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:10:07,659 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:10:08,436 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:10:08,498 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:10:08,721 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:10:08,722 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:10:08,731 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:10:10,499 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:10:13,786 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:10:18,787 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:10:22,391 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:10:23,721 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:10:23,722 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:10:23,762 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:10:23,807 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:10:27,122 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:10:27,123 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:10:27,123 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:10:27,124 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:10:27,507 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:10:28,905 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:10:30,508 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:10:33,906 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:10:38,722 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:10:38,722 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:10:38,731 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:10:39,752 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:10:44,753 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:10:47,233 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:10:47,234 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:10:47,234 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:10:47,235 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:10:47,515 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:10:49,981 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:10:50,516 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:10:52,392 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:10:53,721 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:10:53,722 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:10:53,731 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:10:55,770 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:11:00,771 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:11:05,772 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:11:08,722 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:11:08,722 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:11:08,762 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:11:08,958 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:11:08,959 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:11:08,960 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:11:08,960 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:11:09,524 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:11:11,693 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:11:12,525 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:11:16,694 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:11:21,695 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:11:22,393 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:11:23,722 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:11:23,722 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:11:23,731 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:11:26,843 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:11:29,312 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:11:29,313 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:11:29,313 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:11:29,314 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:11:29,532 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:11:32,159 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:11:32,533 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:11:37,160 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:11:38,722 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:11:38,723 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:11:38,731 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:11:42,814 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:11:47,815 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:11:48,472 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:11:48,473 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:11:48,474 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:11:48,474 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:11:48,539 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:11:50,539 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:11:52,394 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:11:53,395 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:11:53,722 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:11:53,723 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:11:53,732 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:11:58,754 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:12:03,754 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:12:08,722 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:12:08,723 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:12:08,762 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:12:08,841 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:12:08,939 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:12:08,941 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:12:08,941 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:12:08,941 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:12:09,547 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:12:10,548 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:12:14,656 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:12:19,657 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:12:22,395 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:12:23,722 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:12:23,723 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:12:23,732 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:12:24,766 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:12:29,766 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:12:32,701 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:12:32,702 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:12:32,702 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:12:32,702 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:12:33,557 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:12:34,557 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:12:35,437 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:12:38,722 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:12:38,723 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:12:38,732 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:12:40,791 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:12:45,792 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:12:50,793 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:12:52,396 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:12:53,723 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:12:53,723 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:12:53,763 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:12:55,324 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:12:55,325 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:12:55,325 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:12:55,325 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:12:55,565 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:12:56,068 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:12:58,566 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:13:01,069 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:13:06,070 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:13:08,723 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:13:08,724 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:13:08,732 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:13:11,804 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:13:15,356 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:13:15,357 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:13:15,357 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:13:15,358 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:13:15,573 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:13:17,072 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:13:18,574 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:13:22,073 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:13:22,397 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:13:23,723 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:13:23,724 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:13:23,733 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:13:27,786 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:13:32,787 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:13:36,316 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:13:36,317 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:13:36,318 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:13:36,318 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:13:36,581 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:13:38,101 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:13:38,582 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:13:38,726 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:13:38,726 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:13:38,732 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:13:43,831 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:13:48,832 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:13:52,398 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:13:53,725 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:13:53,725 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:13:53,766 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:13:54,796 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:13:59,339 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:13:59,340 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:13:59,341 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:13:59,341 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:13:59,591 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:14:00,053 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:14:02,592 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:14:05,054 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:14:08,729 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:14:08,730 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:14:08,738 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:14:10,835 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:14:15,835 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:14:18,679 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:14:18,681 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:14:18,681 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:14:18,681 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:14:19,599 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:14:20,600 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:14:21,430 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:14:22,399 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:14:23,726 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:14:23,727 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:14:23,733 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:14:26,854 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:14:31,855 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:14:36,856 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:14:38,725 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:14:38,726 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:14:38,766 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:14:39,026 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:14:39,027 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:14:39,027 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:14:39,028 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:14:39,607 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:14:40,607 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:14:42,752 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:14:47,753 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:14:52,404 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:14:53,405 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:14:53,726 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:14:53,727 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:14:53,733 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:14:56,729 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:14:56,730 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:14:56,730 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:14:56,730 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:14:57,615 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:14:58,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:14:58,616 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:15:03,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:15:08,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:15:08,726 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:15:08,727 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:15:08,733 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:15:13,794 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:15:18,795 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:15:20,510 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:15:20,511 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:15:20,512 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:15:20,512 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:15:20,623 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:15:22,401 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:15:22,624 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:15:23,726 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:15:23,727 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:15:23,733 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:15:24,779 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:15:29,779 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:15:34,780 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:15:38,727 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:15:38,727 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:15:38,768 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:15:39,797 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:15:41,400 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:15:41,401 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:15:41,401 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:15:41,403 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:15:41,634 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:15:44,635 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:15:45,122 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:15:50,123 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:15:52,401 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:15:53,726 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:15:53,727 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:15:53,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:15:55,777 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:16:00,777 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:16:02,178 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:16:02,179 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:16:02,179 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:16:02,179 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:16:02,643 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:16:04,643 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:16:05,939 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:16:08,727 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:16:08,728 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:16:08,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:16:11,850 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:16:16,851 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:16:21,851 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:16:22,403 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:16:22,937 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:16:22,938 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:16:22,938 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:16:22,938 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:16:23,650 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:16:23,776 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:16:23,777 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:16:23,777 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:16:24,651 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:16:26,882 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:16:31,883 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:16:36,884 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:16:38,773 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:16:38,773 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:16:38,774 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:16:42,853 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:16:43,648 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:16:43,649 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:16:43,650 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:16:43,650 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:16:43,657 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:16:46,659 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:16:48,357 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:16:52,404 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:16:53,405 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:16:53,773 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:16:53,774 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:16:53,774 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:16:58,840 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:17:03,841 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:17:04,721 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:17:04,722 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:17:04,722 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:17:04,722 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:17:05,667 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:17:06,667 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:17:08,773 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:17:08,773 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:17:08,774 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:17:09,820 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:17:14,821 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:17:19,822 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:17:22,405 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:17:23,772 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:17:23,773 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:17:23,812 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:17:24,886 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:17:26,238 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:17:26,240 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:17:26,240 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:17:26,240 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:17:26,673 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:17:28,674 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:17:30,003 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:17:35,004 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:17:38,772 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:17:38,773 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:17:38,775 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:17:40,838 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:17:45,839 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:17:49,161 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:17:49,163 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:17:49,163 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:17:49,163 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:17:49,681 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:17:50,682 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:17:50,884 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:17:52,406 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:17:53,772 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:17:53,773 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:17:53,777 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:17:56,879 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:18:01,880 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:18:06,881 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:18:08,774 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:18:08,774 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:18:08,815 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:18:09,197 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:18:09,199 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:18:09,199 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:18:09,199 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:18:09,689 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:18:10,690 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:18:11,910 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:18:16,911 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:18:21,912 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:18:22,406 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:18:23,774 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:18:23,775 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:18:23,775 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:18:27,804 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:18:30,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:18:30,569 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:18:30,570 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:18:30,570 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:18:30,697 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:18:32,698 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:18:33,305 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:18:38,306 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:18:38,775 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:18:38,775 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:18:38,776 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:18:43,885 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:18:48,886 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:18:50,779 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:18:50,780 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:18:50,780 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:18:50,780 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:18:51,704 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:18:52,408 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:18:52,705 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:18:53,773 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:18:53,774 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:18:53,776 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:18:54,851 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:18:59,853 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:19:04,853 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:19:08,773 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:19:08,774 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:19:08,813 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:19:10,820 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:19:11,534 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:19:11,536 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:19:11,536 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:19:11,536 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:19:11,712 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:19:12,712 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:19:16,262 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:19:21,263 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:19:22,409 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:19:23,773 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:19:23,774 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:19:23,775 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:19:26,865 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:19:30,023 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:19:30,024 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:19:30,025 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:19:30,025 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:19:30,718 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:19:32,719 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:19:32,751 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:19:37,752 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:19:38,773 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:19:38,774 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:19:38,777 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:19:42,905 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:19:47,906 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:19:52,103 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:19:52,104 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:19:52,104 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:19:52,105 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:19:52,410 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:19:52,726 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:19:53,774 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:19:53,774 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:19:53,775 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:19:53,842 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:19:54,727 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:19:58,843 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:20:03,845 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:20:08,774 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:20:08,774 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:20:08,815 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:20:08,891 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:20:11,854 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:20:11,856 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:20:11,856 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:20:11,857 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:20:12,733 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:20:14,642 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:20:14,734 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:20:19,643 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:20:22,410 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:20:23,774 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:20:23,774 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:20:23,775 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:20:24,807 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:20:29,808 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:20:33,740 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:20:33,741 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:20:33,741 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:20:33,741 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:20:33,742 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:20:35,450 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:20:36,743 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:20:38,774 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:20:38,775 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:20:38,779 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:20:40,816 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:20:45,817 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:20:50,818 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:20:52,411 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:20:53,359 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:20:53,360 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:20:53,360 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:20:53,360 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:20:53,750 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:20:53,806 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:20:53,806 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:20:53,807 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:20:54,750 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:20:56,088 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:21:01,089 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:21:06,090 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:21:08,781 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:21:08,782 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:21:08,782 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:21:11,862 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:21:12,903 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:21:12,904 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:21:12,904 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:21:12,905 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:21:13,757 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:21:14,757 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:21:17,616 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:21:22,412 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:21:23,413 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:21:23,781 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:21:23,781 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:21:23,784 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:21:28,910 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:21:33,911 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:21:35,525 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:21:35,526 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:21:35,526 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:21:35,527 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:21:35,765 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:21:36,765 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:21:38,781 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:21:38,782 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:21:38,783 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:21:39,905 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:21:44,906 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:21:49,907 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:21:52,413 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:21:53,780 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:21:53,780 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:21:53,821 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:21:55,867 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:21:57,665 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:21:57,666 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:21:57,667 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:21:57,667 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:21:57,774 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:21:59,775 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:22:01,417 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:22:06,417 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:22:08,780 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:22:08,781 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:22:08,784 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:22:11,811 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:22:16,812 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:22:17,013 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:22:17,015 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:22:17,015 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:22:17,015 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:22:17,783 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:22:19,784 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:22:22,414 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:22:22,416 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:22:23,780 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:22:23,781 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:22:23,781 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:22:27,894 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:22:32,895 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:22:37,896 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:22:38,780 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:22:38,781 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:22:38,821 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:22:39,284 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:22:39,285 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:22:39,285 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:22:39,286 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:22:39,791 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:22:41,792 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:22:43,014 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:22:48,015 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:22:52,415 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:22:53,416 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:22:53,780 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:22:53,781 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:22:53,784 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:22:58,843 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:22:58,961 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:22:58,962 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:22:58,962 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:22:58,963 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:22:59,798 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:23:01,798 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:23:03,928 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:23:08,780 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:23:08,781 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:23:08,784 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:23:09,908 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:23:14,909 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:23:19,910 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:23:19,950 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:23:19,951 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:23:19,952 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:23:19,952 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:23:20,806 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:23:22,416 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:23:23,783 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:23:23,783 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:23:23,786 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:23:23,807 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:23:25,871 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:23:30,872 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:23:35,873 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:23:38,781 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:23:38,782 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:23:38,824 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:23:41,480 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:23:41,481 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:23:41,481 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:23:41,482 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:23:41,482 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:23:41,813 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:23:43,814 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:23:47,208 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:23:52,209 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:23:52,417 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:23:53,782 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:23:53,783 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:23:53,786 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:23:57,835 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:24:01,516 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:24:01,517 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:24:01,517 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:24:01,518 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:24:01,821 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:24:03,225 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:24:03,822 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:24:08,227 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:24:08,783 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:24:08,784 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:24:08,786 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:24:13,840 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:24:18,840 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:24:21,338 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:24:21,339 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:24:21,339 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:24:21,340 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:24:21,828 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:24:22,418 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:24:23,782 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:24:23,782 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:24:23,785 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:24:23,828 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:24:24,838 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:24:29,839 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:24:34,840 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:24:38,782 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:24:38,782 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:24:38,822 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:24:39,746 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:24:39,748 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:24:39,748 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:24:39,748 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:24:39,834 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:24:40,749 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:24:43,836 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:24:45,865 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:24:50,866 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:24:52,419 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:24:53,782 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:24:53,783 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:24:53,785 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:24:56,810 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:25:01,811 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:25:04,473 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:25:04,474 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:25:04,474 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:25:04,475 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:25:04,845 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:25:07,310 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:25:07,846 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:25:08,782 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:25:08,782 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:25:08,784 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:25:12,898 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:25:17,899 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:25:22,420 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:25:23,421 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:25:23,782 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:25:23,783 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:25:23,823 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:25:28,862 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:25:29,501 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:25:29,502 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:25:29,502 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:25:29,503 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:25:29,854 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:25:31,855 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:25:34,218 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:25:38,782 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:25:38,783 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:25:38,783 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:25:39,871 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:25:44,872 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:25:49,872 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:25:50,961 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:25:50,962 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:25:50,962 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:25:50,963 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:25:51,863 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:25:52,421 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:25:53,782 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:25:53,783 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:25:53,785 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:25:53,863 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:25:55,862 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:26:00,863 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:26:05,864 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:26:08,782 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:26:08,783 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:26:08,823 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:26:10,901 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:26:12,082 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:26:12,083 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:26:12,083 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:26:12,084 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:26:12,870 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:26:13,871 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:26:16,801 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:26:21,802 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:26:22,422 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:26:23,783 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:26:23,783 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:26:23,787 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:26:26,816 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:26:31,817 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:26:33,435 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:26:33,436 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:26:33,436 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:26:33,437 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:26:33,878 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:26:35,879 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:26:37,487 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:26:38,783 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:26:38,784 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:26:38,786 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:26:42,889 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:26:47,890 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:26:52,423 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:26:53,424 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:26:53,783 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:26:53,783 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:26:53,824 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:26:57,299 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:26:57,300 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:26:57,301 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:26:57,301 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:26:57,886 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:26:59,029 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:26:59,887 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:27:04,030 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:27:08,783 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:27:08,783 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:27:08,786 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:27:09,880 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:27:14,881 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:27:16,391 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:27:16,393 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:27:16,393 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:27:16,393 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:27:16,893 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:27:19,894 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:27:20,363 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:27:22,424 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:27:23,783 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:27:23,783 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:27:23,786 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:27:25,876 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:27:30,877 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:27:34,206 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:27:34,207 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:27:34,208 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:27:34,208 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:27:34,899 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:27:35,926 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:27:37,900 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:27:38,783 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:27:38,784 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:27:38,784 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:27:41,909 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:27:46,910 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:27:51,910 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:27:52,424 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:27:53,783 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:27:53,784 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:27:53,824 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:27:57,894 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:27:59,151 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:27:59,152 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:27:59,153 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:27:59,153 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:27:59,907 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:28:01,908 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:28:02,902 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:28:07,902 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:28:08,784 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:28:08,785 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:28:08,785 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:28:13,890 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:28:18,891 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:28:22,121 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:28:22,122 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:28:22,123 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:28:22,123 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:28:22,425 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:28:22,915 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:28:23,784 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:28:23,785 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:28:23,785 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:28:23,916 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:28:24,884 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:28:29,885 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:28:34,886 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:28:38,784 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:28:38,785 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:28:38,826 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:28:40,865 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:28:40,998 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:28:40,999 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:28:40,999 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:28:41,000 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:28:41,922 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:28:43,923 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:28:46,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:28:51,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:28:52,426 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:28:53,784 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:28:53,785 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:28:53,787 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:28:56,862 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:29:01,499 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:29:01,501 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:29:01,501 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:29:01,501 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:29:01,929 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:29:02,231 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:29:03,930 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:29:07,231 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:29:08,784 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:29:08,785 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:29:08,785 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:29:12,865 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:29:17,866 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:29:22,427 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:29:23,428 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:29:23,784 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:29:23,785 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:29:23,825 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:29:23,966 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:29:23,968 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:29:23,968 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:29:23,968 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:29:24,938 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:29:25,939 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:29:28,685 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:29:33,686 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:29:38,687 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:29:38,785 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:29:38,785 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:29:38,788 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:29:43,828 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:29:45,478 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:29:45,479 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:29:45,480 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:29:45,480 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:29:45,946 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:29:47,946 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:29:49,202 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:29:52,428 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:29:53,789 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:29:53,790 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:29:53,790 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:29:54,897 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:29:59,898 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:30:04,899 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:30:06,243 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:30:06,244 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:30:06,244 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:30:06,245 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:30:06,953 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:30:07,954 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:30:08,787 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:30:08,788 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:30:08,788 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:30:10,862 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:30:15,863 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:30:20,864 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:30:22,429 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:30:23,790 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:30:23,790 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:30:23,791 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:30:24,619 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:30:24,620 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:30:24,620 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:30:24,621 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:30:24,961 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:30:26,342 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:30:27,962 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:30:31,343 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:30:36,344 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:30:38,787 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:30:38,788 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:30:38,791 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:30:41,881 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:30:42,419 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:30:42,420 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:30:42,420 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:30:42,421 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:30:42,967 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:30:45,968 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:30:47,172 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:30:52,173 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:30:52,430 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:30:53,788 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:30:53,788 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:30:53,791 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:30:57,872 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:31:01,217 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:31:01,219 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:31:01,219 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:31:01,219 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:31:01,974 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:31:02,931 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:31:03,975 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:31:07,932 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:31:08,790 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:31:08,791 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:31:08,831 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:31:13,881 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:31:18,882 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:31:20,769 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:31:20,770 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:31:20,771 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:31:20,771 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:31:20,982 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:31:22,431 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:31:23,788 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:31:23,789 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:31:23,789 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:31:23,983 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:31:24,835 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:31:29,836 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:31:34,837 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:31:38,788 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:31:38,789 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:31:38,829 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:31:39,855 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:31:41,471 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:31:41,472 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:31:41,472 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:31:41,473 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:31:41,989 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:31:43,990 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:31:45,220 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:31:50,221 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:31:52,432 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:31:53,789 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:31:53,789 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:31:53,792 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:31:55,913 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:32:00,914 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:32:01,862 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:32:01,864 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:32:01,864 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:32:01,864 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:32:01,997 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:32:03,998 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:32:06,593 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:32:08,789 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:32:08,789 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:32:08,792 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:32:11,896 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:32:16,898 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:32:21,898 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:32:22,432 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:32:23,026 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:32:23,028 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:32:23,028 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:32:23,028 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:32:23,874 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:32:23,876 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:32:23,876 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:32:24,006 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:32:26,007 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:32:26,956 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:32:31,957 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:32:36,958 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:32:38,870 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:32:38,871 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:32:38,910 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:32:41,972 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:32:43,393 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:32:43,394 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:32:43,394 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:32:43,394 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:32:44,014 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:32:46,015 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:32:47,157 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:32:52,158 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:32:52,433 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:32:53,869 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:32:53,870 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:32:53,910 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:32:58,007 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:33:03,008 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:33:03,670 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:33:03,671 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:33:03,671 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:33:03,671 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:33:04,021 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:33:06,022 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:33:08,503 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:33:08,869 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:33:08,869 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:33:08,870 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:33:13,959 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:33:18,960 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:33:22,435 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:33:23,869 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:33:23,870 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:33:23,870 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:33:24,921 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:33:24,988 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:33:24,989 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:33:24,989 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:33:24,990 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:33:25,029 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:33:28,030 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:33:30,716 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:33:35,717 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:33:38,869 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:33:38,870 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:33:38,870 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:33:40,995 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:33:45,995 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:33:47,457 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:33:47,458 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:33:47,458 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:33:47,459 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:33:48,037 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:33:50,038 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:33:51,506 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:33:52,435 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:33:53,869 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:33:53,870 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:33:53,870 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:33:56,983 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:34:01,984 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:34:06,985 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:34:08,872 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:34:08,872 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:34:08,873 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:34:10,685 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:34:10,686 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:34:10,686 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:34:10,686 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:34:11,046 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:34:12,046 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:34:12,401 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:34:17,401 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:34:22,402 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:34:22,436 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:34:23,870 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:34:23,871 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:34:23,871 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:34:28,002 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:34:32,204 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:34:32,206 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:34:32,206 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:34:32,206 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:34:33,005 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:34:33,054 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:34:34,055 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:34:38,006 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:34:38,870 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:34:38,870 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:34:38,871 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:34:43,017 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:34:48,018 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:34:52,437 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:34:53,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:34:53,870 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:34:53,871 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:34:53,871 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:34:54,884 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:34:54,885 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:34:54,885 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:34:54,886 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:34:55,064 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:34:58,065 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:34:58,602 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:35:03,603 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:35:08,604 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:35:08,870 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:35:08,871 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:35:08,871 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:35:13,933 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:35:16,460 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:35:16,461 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:35:16,461 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:35:16,461 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:35:17,073 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:35:18,074 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:35:19,226 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:35:22,438 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:35:23,870 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:35:23,871 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:35:23,872 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:35:24,918 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:35:29,919 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:35:34,920 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:35:37,675 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:35:37,676 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:35:37,676 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:35:37,676 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:35:38,082 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:35:38,871 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:35:38,871 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:35:38,872 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:35:40,083 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:35:40,910 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:35:45,911 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:35:50,912 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:35:52,439 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:35:53,871 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:35:53,871 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:35:53,871 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:35:56,906 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:35:58,507 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:35:58,508 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:35:58,508 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:35:58,509 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:35:59,089 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:36:00,090 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:36:02,248 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:36:07,249 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:36:08,871 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:36:08,872 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:36:08,873 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:36:12,946 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:36:17,947 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:36:18,531 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:36:18,532 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:36:18,532 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:36:18,533 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:36:19,096 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:36:20,097 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:36:22,440 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:36:23,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:36:23,871 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:36:23,871 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:36:23,872 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:36:28,927 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:36:33,928 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:36:38,871 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:36:38,872 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:36:38,872 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:36:38,935 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:36:40,610 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:36:40,611 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:36:40,612 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:36:40,612 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:36:41,104 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:36:42,105 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:36:44,604 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:36:49,605 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:36:52,441 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:36:53,871 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:36:53,872 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:36:53,872 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:36:54,939 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:36:59,940 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:37:01,749 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:37:01,750 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:37:01,750 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:37:01,751 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:37:02,111 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:37:04,112 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:37:05,553 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:37:08,871 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:37:08,872 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:37:08,872 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:37:10,895 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:37:15,896 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:37:20,896 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:37:21,929 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:37:21,930 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:37:21,930 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:37:21,931 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:37:22,119 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:37:22,441 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:37:23,872 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:37:23,872 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:37:23,873 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:37:24,120 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:37:25,923 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:37:30,924 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:37:35,925 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:37:38,873 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:37:38,874 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:37:38,874 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:37:40,963 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:37:43,340 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:37:43,342 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:37:43,342 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:37:43,342 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:37:44,127 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:37:46,086 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:37:46,128 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:37:51,087 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:37:52,442 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:37:53,872 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:37:53,873 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:37:53,873 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:37:56,914 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:38:01,915 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:38:03,992 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:38:03,993 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:38:03,994 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:38:03,994 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:38:04,135 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:38:06,135 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:38:07,801 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:38:08,872 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:38:08,872 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:38:08,873 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:38:12,940 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:38:17,941 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:38:22,444 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:38:23,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:38:23,872 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:38:23,873 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:38:23,873 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:38:27,690 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:38:27,691 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:38:27,691 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:38:27,692 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:38:28,143 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:38:29,435 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:38:30,144 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:38:34,437 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:38:38,872 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:38:38,872 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:38:38,873 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:38:39,906 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:38:44,907 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:38:48,011 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:38:48,012 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:38:48,012 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:38:48,013 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:38:48,150 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:38:50,151 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:38:50,751 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:38:52,445 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:38:53,872 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:38:53,873 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:38:53,873 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:38:55,993 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:39:00,993 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:39:05,995 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:39:07,871 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:39:07,872 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:39:07,872 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:39:07,873 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:39:08,158 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:39:08,872 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:39:08,873 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:39:08,874 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:39:10,159 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:39:11,950 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:39:16,951 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:39:21,951 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:39:22,445 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:39:23,873 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:39:23,873 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:39:23,873 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:39:27,917 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:39:31,079 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:39:31,080 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:39:31,081 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:39:31,081 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:39:31,167 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:39:33,008 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:39:34,168 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:39:38,009 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:39:38,881 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:39:38,881 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:39:38,882 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:39:44,003 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:39:49,004 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:39:51,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:39:51,445 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:39:51,445 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:39:51,445 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:39:52,175 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:39:52,446 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:39:53,881 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:39:53,882 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:39:53,883 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:39:54,176 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:39:54,930 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:39:59,931 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:40:04,932 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:40:08,880 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:40:08,881 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:40:08,881 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:40:10,017 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:40:13,510 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:40:13,512 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:40:13,512 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:40:13,512 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:40:14,183 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:40:15,238 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:40:16,183 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:40:20,239 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:40:22,447 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:40:23,879 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:40:23,880 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:40:23,880 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:40:25,950 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:40:30,950 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:40:33,132 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:40:33,134 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:40:33,134 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:40:33,134 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:40:33,189 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:40:36,146 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:40:36,190 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:40:38,879 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:40:38,879 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:40:38,880 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:40:42,009 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:40:47,010 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:40:52,011 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:40:52,448 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:40:52,658 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:40:52,659 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:40:52,660 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:40:52,660 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:40:53,196 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:40:53,879 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:40:53,879 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:40:53,880 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:40:54,196 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:40:57,950 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:41:02,951 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:41:07,952 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:41:08,879 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:41:08,879 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:41:08,880 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:41:13,008 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:41:14,166 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:41:14,167 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:41:14,167 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:41:14,168 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:41:14,203 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:41:16,204 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:41:18,882 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:41:22,449 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:41:23,879 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:41:23,879 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:41:23,880 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:41:23,933 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:41:28,934 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:41:33,935 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:41:37,891 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:41:37,892 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:41:37,892 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:41:37,893 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:41:38,213 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:41:38,879 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:41:38,879 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:41:38,880 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:41:39,020 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:41:40,213 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:41:44,021 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:41:49,022 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:41:52,450 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:41:53,880 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:41:53,880 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:41:53,880 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:41:54,912 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:41:59,913 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:42:00,402 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:42:00,403 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:42:00,403 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:42:00,404 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:42:01,221 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:42:02,221 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:42:05,132 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:42:08,879 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:42:08,880 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:42:08,880 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:42:10,909 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:42:15,910 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:42:20,911 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:42:21,285 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:42:21,287 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:42:21,287 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:42:21,287 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:42:22,228 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:42:22,228 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:42:22,451 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:42:23,880 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:42:23,880 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:42:23,881 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:42:25,911 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:42:30,912 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:42:35,913 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:42:38,880 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:42:38,881 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:42:38,882 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:42:41,003 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:42:41,990 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:42:41,991 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:42:41,992 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:42:41,992 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:42:42,236 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:42:44,237 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:42:46,749 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:42:51,750 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:42:52,452 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:42:53,880 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:42:53,880 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:42:53,881 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:42:56,973 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:43:01,974 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:43:03,369 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:43:03,370 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:43:03,370 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:43:03,370 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:43:04,244 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:43:04,244 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:43:07,082 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:43:08,880 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:43:08,881 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:43:08,881 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:43:12,965 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:43:17,966 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:43:22,454 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:43:23,455 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:43:23,880 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:43:23,881 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:43:23,881 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:43:25,611 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:43:25,613 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:43:25,613 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:43:25,613 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:43:26,251 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:43:28,252 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:43:29,329 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:43:34,330 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:43:38,880 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:43:38,881 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:43:38,882 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:43:39,956 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:43:44,957 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:43:49,958 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:43:50,601 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:43:50,602 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:43:50,602 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:43:50,603 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:43:51,262 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:43:52,455 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:43:53,263 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:43:53,881 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:43:53,881 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:43:53,882 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:43:54,986 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:43:59,987 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:44:04,988 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:44:08,882 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:44:08,882 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:44:08,923 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:44:10,010 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:44:12,591 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:44:12,592 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:44:12,592 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:44:12,593 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:44:13,270 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:44:15,271 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:44:15,290 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:44:20,291 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:44:22,456 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:44:23,881 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:44:23,881 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:44:23,882 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:44:25,942 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:44:30,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:44:33,675 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:44:33,676 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:44:33,677 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:44:33,677 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:44:34,279 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:44:36,374 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:44:37,280 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:44:38,886 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:44:38,887 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:44:38,887 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:44:41,913 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:44:46,915 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:44:51,916 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:44:52,456 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:44:53,655 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:44:53,657 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:44:53,657 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:44:53,657 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:44:54,088 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:44:54,089 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:44:54,091 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:44:54,286 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:44:57,287 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:44:57,370 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:45:02,370 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:45:07,371 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:45:09,051 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:45:09,051 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:45:09,088 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:45:13,074 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:45:13,606 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:45:13,608 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:45:13,608 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:45:13,608 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:45:14,294 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:45:17,295 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:45:18,384 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:45:22,457 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:45:23,458 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:45:24,051 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:45:24,052 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:45:24,088 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:45:29,105 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:45:34,106 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:45:37,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:45:37,432 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:45:37,432 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:45:37,433 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:45:38,305 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:45:39,051 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:45:39,052 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:45:39,089 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:45:39,168 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:45:39,305 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:45:44,169 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:45:49,170 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:45:52,458 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:45:54,052 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:45:54,052 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:45:54,093 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:45:55,123 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:45:59,455 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:45:59,456 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:45:59,456 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:45:59,457 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:46:00,154 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:46:00,312 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:46:01,313 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:46:05,154 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:46:09,052 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:46:09,053 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:46:09,089 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:46:10,180 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:46:15,181 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:46:19,410 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:46:19,411 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:46:19,411 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:46:19,412 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:46:20,320 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:46:21,161 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:46:21,321 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:46:22,459 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:46:24,052 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:46:24,052 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:46:24,089 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:46:27,125 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:46:32,126 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:46:37,127 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:46:39,052 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:46:39,052 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:46:39,089 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:46:39,459 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:46:39,461 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:46:39,461 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:46:39,461 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:46:40,328 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:46:41,328 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:46:42,181 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:46:47,182 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:46:52,182 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:46:52,460 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:46:54,052 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:46:54,053 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:46:54,089 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:46:58,143 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:46:59,937 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:46:59,939 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:46:59,939 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:46:59,939 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:47:00,335 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:47:03,336 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:47:03,658 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:47:08,659 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:47:09,052 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:47:09,053 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:47:09,089 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:47:14,208 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:47:19,209 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:47:22,461 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:47:24,052 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:47:24,053 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:47:24,093 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:47:25,135 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:47:25,348 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:47:25,349 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:47:25,350 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:47:25,350 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:47:26,345 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:47:27,346 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:47:31,064 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:47:36,065 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:47:39,053 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:47:39,053 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:47:39,090 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:47:41,150 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:47:44,594 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:47:44,596 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:47:44,596 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:47:44,596 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:47:45,353 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:47:46,387 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:47:47,354 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:47:51,388 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:47:52,466 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:47:54,052 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:47:54,053 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:47:54,090 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:47:57,147 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:48:02,147 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:48:03,989 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:48:03,990 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:48:03,991 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:48:03,991 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:48:04,361 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:48:07,362 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:48:07,725 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:48:09,053 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:48:09,053 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:48:09,090 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:48:13,093 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:48:18,094 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:48:22,462 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:48:23,463 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:48:24,053 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:48:24,054 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:48:24,094 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:48:25,689 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:48:25,690 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:48:25,690 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:48:25,692 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:48:26,368 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:48:28,802 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:48:29,370 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:48:33,803 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:48:38,804 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:48:39,053 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:48:39,053 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:48:39,090 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:48:44,131 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:48:49,132 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:48:52,264 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:48:52,265 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:48:52,266 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:48:52,266 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:48:52,379 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:48:52,463 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:48:54,053 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:48:54,054 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:48:54,090 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:48:55,102 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:48:55,380 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:49:00,103 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:49:05,104 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:49:09,053 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:49:09,054 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:49:09,094 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:49:10,168 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:49:12,925 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:49:12,927 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:49:12,927 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:49:12,929 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:49:13,386 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:49:15,387 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:49:15,641 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:49:20,642 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:49:22,464 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:49:24,055 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:49:24,056 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:49:24,091 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:49:26,170 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:49:31,171 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:49:32,921 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:49:32,922 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:49:32,922 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:49:32,922 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:49:33,393 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:49:35,394 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:49:36,654 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:49:39,054 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:49:39,055 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:49:39,091 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:49:42,170 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:49:47,171 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:49:52,172 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:49:52,437 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:49:52,438 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:49:52,439 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:49:52,441 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:49:52,465 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:49:53,401 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:49:54,054 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:49:54,055 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:49:54,091 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:49:55,402 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:49:57,173 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:50:02,174 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:50:07,175 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:50:09,054 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:50:09,055 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:50:09,095 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:50:13,099 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:50:18,100 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:50:18,665 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:50:18,666 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:50:18,666 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:50:18,667 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:50:19,411 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:50:21,412 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:50:22,466 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:50:23,466 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:50:24,054 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:50:24,055 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:50:24,091 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:50:29,171 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:50:34,172 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:50:37,915 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:50:37,916 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:50:37,916 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:50:37,916 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:50:38,420 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:50:39,054 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:50:39,055 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:50:39,092 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:50:39,420 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:50:40,157 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:50:45,158 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:50:50,159 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:50:52,467 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:50:54,055 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:50:54,055 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:50:54,096 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:50:56,101 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:50:57,590 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:50:57,591 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:50:57,592 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:50:57,592 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:50:58,427 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:50:59,428 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:51:01,356 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:51:06,357 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:51:09,055 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:51:09,056 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:51:09,092 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:51:12,157 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:51:17,158 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:51:18,297 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:51:18,298 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:51:18,299 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:51:18,299 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:51:18,435 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:51:21,437 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:51:22,468 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:51:22,469 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:51:24,055 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:51:24,056 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:51:24,092 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:51:28,190 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:51:33,191 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:51:38,191 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:51:39,055 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:51:39,056 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:51:39,096 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:51:41,500 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:51:41,501 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:51:41,502 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:51:41,502 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:51:42,445 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:51:43,214 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:51:43,445 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:51:48,215 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:51:52,469 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:51:53,469 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:51:54,055 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:51:54,056 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:51:54,092 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:51:59,174 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:52:02,212 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:52:02,214 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:52:02,214 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:52:02,215 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:52:02,452 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:52:04,956 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:52:05,453 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:52:09,055 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:52:09,056 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:52:09,092 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:52:10,174 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:52:15,174 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:52:20,175 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:52:21,117 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:52:21,118 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:52:21,119 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:52:21,119 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:52:21,460 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:52:22,470 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:52:23,461 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:52:24,057 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:52:24,057 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:52:24,092 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:52:26,159 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:52:31,160 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:52:36,161 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:52:39,056 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:52:39,056 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:52:39,097 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:52:41,507 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:52:41,508 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:52:41,509 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:52:41,509 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:52:41,510 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:52:42,468 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:52:43,468 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:52:47,227 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:52:52,228 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:52:52,471 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:52:54,056 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:52:54,057 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:52:54,093 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:52:58,169 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:53:03,170 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:53:03,787 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:53:03,788 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:53:03,789 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:53:03,789 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:53:04,479 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:53:05,480 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:53:08,532 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:53:09,058 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:53:09,058 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:53:09,093 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:53:14,131 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:53:19,133 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:53:22,472 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:53:23,546 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:53:23,547 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:53:23,547 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:53:23,547 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:53:24,281 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:53:24,285 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:53:24,286 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:53:24,287 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:53:24,487 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:53:25,488 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:53:29,345 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:53:34,346 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:53:39,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:53:39,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:53:39,278 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:53:39,401 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:53:44,401 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:53:46,062 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:53:46,063 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:53:46,064 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:53:46,064 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:53:46,497 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:53:47,497 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:53:49,817 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:53:52,473 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:53:54,277 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:53:54,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:53:54,279 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:53:55,356 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:54:00,357 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:54:05,358 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:54:06,827 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:54:06,828 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:54:06,828 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:54:06,829 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:54:07,505 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:54:09,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:54:09,279 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:54:09,279 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:54:09,505 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:54:10,398 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:54:15,399 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:54:20,400 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:54:22,474 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:54:24,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:54:24,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:54:24,279 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:54:26,385 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:54:28,947 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:54:28,948 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:54:28,948 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:54:28,949 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:54:29,513 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:54:31,514 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:54:31,744 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:54:36,745 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:54:39,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:54:39,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:54:39,279 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:54:42,309 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:54:47,310 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:54:50,709 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:54:50,710 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:54:50,711 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:54:50,711 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:54:51,523 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:54:52,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:54:52,476 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:54:53,524 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:54:54,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:54:54,279 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:54:54,279 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:54:58,326 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:55:03,327 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:55:08,328 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:55:09,059 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:55:09,060 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:55:09,060 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:55:09,060 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:55:09,497 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:55:09,497 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:55:09,498 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:55:09,529 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:55:11,531 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:55:13,789 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:55:18,790 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:55:22,476 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:55:24,458 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:55:24,460 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:55:24,461 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:55:24,499 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:55:29,501 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:55:30,916 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:55:30,918 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:55:30,918 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:55:30,918 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:55:31,539 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:55:33,539 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:55:34,786 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:55:39,459 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:55:39,462 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:55:39,462 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:55:40,522 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:55:45,522 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:55:50,523 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:55:51,097 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:55:51,098 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:55:51,099 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:55:51,099 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:55:51,546 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:55:52,477 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:55:53,547 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:55:54,472 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:55:54,473 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:55:54,476 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:55:55,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:56:00,574 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:56:05,575 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:56:09,461 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:56:09,461 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:56:09,462 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:56:10,061 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:56:10,067 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:56:10,067 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:56:10,068 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:56:10,554 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:56:10,800 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:56:13,555 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:56:15,801 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:56:20,802 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:56:22,478 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:56:24,461 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:56:24,463 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:56:24,463 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:56:26,580 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:56:28,418 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:56:28,420 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:56:28,420 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:56:28,420 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:56:28,561 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:56:31,562 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:56:32,148 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:56:37,148 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:56:39,461 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:56:39,462 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:56:39,462 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:56:42,582 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:56:46,963 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:56:46,965 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:56:46,965 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:56:46,965 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:56:47,573 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:56:47,695 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:56:49,574 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:56:52,479 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:56:53,480 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:56:54,461 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:56:54,463 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:56:54,463 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:56:58,541 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:57:03,542 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:57:08,543 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:57:08,972 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:57:08,973 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:57:08,974 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:57:08,974 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:57:09,581 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:57:09,709 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:57:09,710 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:57:09,749 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:57:11,582 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:57:13,765 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:57:18,766 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:57:22,480 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:57:24,480 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:57:24,705 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:57:24,705 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:57:24,745 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:57:29,822 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:57:31,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:57:31,440 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:57:31,440 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:57:31,441 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:57:31,589 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:57:33,591 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:57:35,156 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:57:39,705 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:57:39,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:57:39,706 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:57:40,826 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:57:45,827 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:57:49,472 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:57:49,474 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:57:49,474 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:57:49,474 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:57:49,597 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:57:51,354 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:57:51,597 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:57:52,480 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:57:54,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:57:54,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:57:54,707 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:57:56,824 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:58:01,825 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:58:06,826 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:58:09,705 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:58:09,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:58:09,707 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:58:11,357 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:58:11,358 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:58:11,358 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:58:11,359 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:58:11,605 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:58:12,091 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:58:13,606 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:58:17,092 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:58:22,093 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:58:22,481 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:58:24,708 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:58:24,709 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:58:24,709 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:58:27,743 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:58:31,048 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:58:31,049 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:58:31,049 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:58:31,050 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:58:31,613 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:58:32,949 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:58:33,614 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:58:37,950 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:58:39,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:58:39,707 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:58:39,710 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:58:43,783 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:58:48,784 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:58:52,482 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:58:54,253 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:58:54,254 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:58:54,255 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:58:54,255 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:58:54,256 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:58:54,622 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:58:54,705 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:58:54,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:58:54,707 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:58:57,623 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:58:59,982 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:59:04,983 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:59:09,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:59:09,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:59:09,707 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:59:10,824 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:59:14,184 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:59:14,185 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:59:14,186 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:59:14,186 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:59:14,631 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:59:15,915 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:59:17,632 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:59:20,915 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:59:22,483 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:59:24,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:59:24,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:59:24,707 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:59:26,738 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:59:31,739 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:59:34,612 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:59:34,613 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:59:34,613 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:59:34,614 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:59:34,639 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:59:37,364 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:59:37,640 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 12:59:39,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:59:39,708 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:59:39,708 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:59:42,773 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:59:47,774 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:59:52,484 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 12:59:53,485 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:59:54,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 12:59:54,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 12:59:54,707 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 12:59:57,332 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 12:59:57,334 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 12:59:57,334 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 12:59:57,334 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 12:59:57,648 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 12:59:59,058 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 12:59:59,649 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:00:04,059 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:00:09,060 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:00:09,708 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:00:09,709 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:00:09,709 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:00:14,847 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:00:15,676 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:00:15,677 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:00:15,678 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:00:15,678 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:00:16,656 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:00:17,657 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:00:20,389 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:00:22,485 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:00:24,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:00:24,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:00:24,707 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:00:25,780 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:00:30,781 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:00:35,719 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:00:35,720 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:00:35,721 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:00:35,721 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:00:36,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:00:36,665 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:00:37,665 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:00:39,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:00:39,708 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:00:39,708 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:00:41,762 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:00:46,763 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:00:51,764 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:00:52,487 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:00:54,709 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:00:54,709 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:00:54,709 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:00:56,457 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:00:56,459 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:00:56,459 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:00:56,460 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:00:56,672 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:00:57,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:00:59,673 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:01:02,279 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:01:07,281 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:01:09,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:01:09,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:01:09,708 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:01:12,765 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:01:16,081 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:01:16,082 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:01:16,082 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:01:16,082 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:01:16,679 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:01:17,679 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:01:17,928 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:01:22,488 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:01:23,489 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:01:24,712 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:01:24,712 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:01:24,713 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:01:28,833 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:01:33,834 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:01:37,969 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:01:37,970 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:01:37,970 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:01:37,970 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:01:38,687 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:01:39,687 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:01:39,694 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:01:39,709 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:01:39,709 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:01:39,709 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:01:44,740 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:01:49,741 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:01:52,488 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:01:54,710 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:01:54,710 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:01:54,711 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:01:54,831 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:01:58,975 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:01:58,976 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:01:58,976 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:01:58,978 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:01:59,694 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:02:00,719 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:02:01,695 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:02:05,720 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:02:09,710 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:02:09,710 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:02:09,711 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:02:10,752 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:02:15,753 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:02:19,675 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:02:19,676 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:02:19,676 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:02:19,677 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:02:19,702 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:02:21,700 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:02:21,703 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:02:22,489 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:02:24,709 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:02:24,709 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:02:24,710 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:02:26,836 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:02:31,837 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:02:36,838 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:02:39,710 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:02:39,711 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:02:39,711 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:02:41,806 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:02:41,808 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:02:41,808 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:02:41,808 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:02:42,533 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:02:42,712 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:02:43,713 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:02:47,535 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:02:52,490 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:02:53,491 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:02:54,709 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:02:54,710 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:02:54,711 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:02:58,830 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:03:03,488 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:03:03,490 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:03:03,490 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:03:03,490 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:03:03,719 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:03:04,210 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:03:05,720 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:03:09,211 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:03:09,711 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:03:09,712 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:03:09,715 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:03:14,769 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:03:19,770 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:03:22,261 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:03:22,262 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:03:22,262 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:03:22,263 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:03:22,491 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:03:22,727 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:03:23,727 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:03:24,710 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:03:24,711 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:03:24,712 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:03:24,797 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:03:29,798 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:03:34,799 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:03:39,710 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:03:39,711 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:03:39,751 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:03:40,767 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:03:42,202 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:03:42,203 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:03:42,204 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:03:42,204 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:03:42,734 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:03:43,735 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:03:45,938 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:03:50,939 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:03:52,492 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:03:54,710 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:03:54,711 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:03:54,714 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:03:56,821 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:04:01,669 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:04:01,671 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:04:01,671 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:04:01,672 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:04:01,742 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:04:02,404 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:04:03,743 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:04:07,405 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:04:09,710 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:04:09,711 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:04:09,714 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:04:12,841 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:04:17,842 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:04:22,497 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:04:22,689 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:04:22,690 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:04:22,691 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:04:22,691 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:04:22,751 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:04:23,414 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:04:24,711 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:04:24,711 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:04:24,712 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:04:25,752 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:04:28,762 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:04:33,763 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:04:38,763 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:04:39,710 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:04:39,711 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:04:39,751 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:04:43,831 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:04:44,315 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:04:44,316 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:04:44,317 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:04:44,317 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:04:44,759 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:04:45,759 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:04:49,064 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:04:52,494 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:04:54,495 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:04:54,710 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:04:54,711 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:04:54,712 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:04:59,767 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:05:04,768 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:05:06,766 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:05:06,767 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:05:06,767 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:05:06,770 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:05:06,770 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:05:09,710 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:05:09,711 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:05:09,713 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:05:09,768 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:05:09,771 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:05:14,770 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:05:19,771 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:05:22,495 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:05:24,711 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:05:24,711 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:05:24,748 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:05:25,749 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:05:30,381 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:05:30,382 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:05:30,382 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:05:30,382 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:05:30,778 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:05:31,236 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:05:31,779 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:05:36,237 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:05:39,711 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:05:39,712 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:05:39,712 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:05:41,791 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:05:46,792 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:05:51,680 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:05:51,681 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:05:51,682 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:05:51,684 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:05:51,786 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:05:52,496 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:05:52,496 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:05:53,787 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:05:54,711 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:05:54,711 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:05:54,715 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:05:57,737 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:06:02,738 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:06:07,739 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:06:09,711 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:06:09,712 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:06:09,751 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:06:12,266 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:06:12,267 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:06:12,267 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:06:12,267 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:06:12,793 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:06:13,042 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:06:13,794 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:06:18,043 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:06:22,496 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:06:23,497 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:06:24,711 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:06:24,712 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:06:24,715 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:06:28,800 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:06:31,373 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:06:31,374 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:06:31,374 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:06:31,375 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:06:31,800 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:06:33,801 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:06:34,146 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:06:39,147 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:06:39,711 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:06:39,712 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:06:39,715 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:06:44,746 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:06:49,747 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:06:52,498 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:06:54,059 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:06:54,061 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:06:54,061 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:06:54,061 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:06:54,764 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:06:54,765 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:06:54,766 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:06:54,809 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:06:54,885 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:06:55,810 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:06:59,886 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:07:04,887 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:07:09,759 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:07:09,760 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:07:09,763 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:07:09,893 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:07:14,894 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:07:16,250 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:07:16,251 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:07:16,251 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:07:16,252 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:07:16,819 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:07:18,820 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:07:20,013 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:07:22,499 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:07:24,759 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:07:24,760 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:07:24,763 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:07:25,831 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:07:30,832 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:07:35,833 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:07:39,504 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:07:39,505 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:07:39,505 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:07:39,506 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:07:39,828 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:07:39,965 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:07:39,965 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:07:39,966 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:07:41,252 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:07:42,829 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:07:46,252 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:07:51,253 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:07:52,500 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:07:54,941 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:07:54,941 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:07:54,981 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:07:56,993 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:08:00,501 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:08:00,502 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:08:00,502 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:08:00,502 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:08:00,837 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:08:02,231 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:08:02,838 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:08:07,232 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:08:09,941 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:08:09,941 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:08:09,978 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:08:12,978 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:08:17,979 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:08:22,501 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:08:23,349 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:08:23,350 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:08:23,350 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:08:23,351 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:08:23,351 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:08:23,847 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:08:24,941 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:08:24,941 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:08:24,981 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:08:26,848 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:08:28,992 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:08:33,993 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:08:38,994 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:08:39,941 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:08:39,941 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:08:39,981 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:08:42,791 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:08:42,792 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:08:42,793 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:08:42,793 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:08:42,854 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:08:44,676 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:08:46,856 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:08:49,677 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:08:52,502 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:08:54,926 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:08:54,941 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:08:54,941 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:08:54,981 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:09:00,057 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:09:04,392 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:09:04,393 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:09:04,394 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:09:04,394 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:09:04,862 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:09:05,110 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:09:06,863 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:09:09,942 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:09:09,942 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:09:09,982 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:09:11,055 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:09:16,056 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:09:21,057 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:09:22,503 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:09:24,941 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:09:24,942 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:09:24,983 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:09:25,524 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:09:25,525 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:09:25,525 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:09:25,526 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:09:25,870 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:09:26,241 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:09:28,871 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:09:31,243 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:09:36,244 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:09:39,942 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:09:39,942 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:09:39,982 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:09:42,081 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:09:45,966 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:09:45,968 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:09:45,968 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:09:45,968 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:09:46,878 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:09:47,697 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:09:48,879 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:09:52,504 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:09:53,505 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:09:54,942 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:09:54,942 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:09:54,983 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:09:59,046 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:10:04,046 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:10:08,731 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:10:08,732 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:10:08,732 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:10:08,732 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:10:08,887 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:10:09,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:10:09,942 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:10:09,943 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:10:09,982 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:10:10,888 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:10:15,070 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:10:20,071 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:10:22,506 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:10:24,942 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:10:24,943 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:10:24,981 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:10:25,981 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:10:30,982 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:10:32,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:10:32,735 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:10:32,736 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:10:32,736 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:10:32,897 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:10:34,898 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:10:36,469 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:10:39,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:10:39,943 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:10:39,984 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:10:42,006 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:10:47,006 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:10:52,007 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:10:52,506 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:10:53,752 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:10:53,753 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:10:53,753 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:10:53,754 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:10:53,905 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:10:54,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:10:54,943 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:10:54,984 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:10:56,907 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:10:58,004 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:11:03,005 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:11:08,006 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:11:09,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:11:09,943 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:11:09,983 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:11:13,009 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:11:14,389 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:11:14,390 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:11:14,390 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:11:14,391 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:11:14,913 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:11:16,914 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:11:18,284 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:11:22,507 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:11:23,508 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:11:24,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:11:24,944 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:11:24,983 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:11:28,986 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:11:33,987 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:11:35,196 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:11:35,197 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:11:35,197 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:11:35,198 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:11:35,921 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:11:38,922 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:11:39,921 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:11:39,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:11:39,943 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:11:39,983 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:11:45,027 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:11:50,028 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:11:52,508 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:11:54,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:11:54,944 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:11:54,983 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:11:55,261 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:11:55,263 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:11:55,263 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:11:55,263 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:11:55,264 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:11:55,929 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:11:58,931 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:12:00,998 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:12:05,998 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:12:09,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:12:09,944 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:12:09,984 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:12:11,007 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:12:16,008 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:12:18,396 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:12:18,397 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:12:18,398 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:12:18,398 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:12:18,939 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:12:20,940 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:12:21,124 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:12:22,509 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:12:24,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:12:24,944 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:12:24,972 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:12:26,972 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:12:31,973 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:12:36,974 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:12:39,495 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:12:39,500 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:12:39,501 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:12:39,501 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:12:39,948 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:12:40,069 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:12:40,069 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:12:40,070 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:12:42,348 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:12:42,949 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:12:47,349 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:12:52,350 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:12:52,510 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:12:55,028 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:12:55,029 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:12:55,030 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:12:58,066 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:12:59,733 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:12:59,734 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:12:59,734 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:12:59,735 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:12:59,957 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:13:02,958 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:13:03,456 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:13:08,457 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:13:10,027 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:13:10,027 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:13:10,028 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:13:14,099 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:13:19,100 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:13:22,511 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:13:23,262 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:13:23,263 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:13:23,263 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:13:23,264 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:13:23,966 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:13:25,027 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:13:25,028 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:13:25,028 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:13:25,151 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:13:26,967 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:13:30,152 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:13:35,153 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:13:40,027 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:13:40,029 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:13:40,029 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:13:41,117 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:13:46,118 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:13:46,686 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:13:46,687 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:13:46,687 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:13:46,688 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:13:46,976 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:13:48,976 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:13:51,393 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:13:52,511 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:13:55,027 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:13:55,028 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:13:55,029 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:13:57,060 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:14:02,061 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:14:05,450 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:14:05,451 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:14:05,451 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:14:05,452 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:14:05,983 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:14:07,190 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:14:08,984 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:14:10,028 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:14:10,028 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:14:10,029 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:14:13,124 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:14:18,125 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:14:22,512 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:14:23,513 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:14:24,924 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:14:24,925 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:14:24,926 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:14:24,926 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:14:24,990 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:14:25,028 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:14:25,028 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:14:25,029 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:14:28,992 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:14:29,026 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:14:34,027 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:14:39,028 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:14:40,030 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:14:40,031 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:14:40,031 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:14:44,103 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:14:47,640 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:14:47,641 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:14:47,642 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:14:47,642 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:14:47,999 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:14:49,364 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:14:51,001 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:14:52,513 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:14:54,514 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:14:55,028 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:14:55,029 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:14:55,029 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:15:00,063 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:15:05,064 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:15:06,053 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:15:06,054 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:15:06,054 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:15:06,055 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:15:07,007 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:15:09,008 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:15:10,028 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:15:10,029 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:15:10,029 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:15:10,128 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:15:15,129 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:15:20,130 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:15:22,514 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:15:25,028 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:15:25,030 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:15:25,031 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:15:26,098 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:15:27,327 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:15:27,328 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:15:27,328 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:15:27,329 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:15:28,015 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:15:31,016 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:15:31,105 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:15:36,106 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:15:40,028 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:15:40,029 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:15:40,030 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:15:42,080 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:15:47,081 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:15:48,189 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:15:48,190 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:15:48,190 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:15:48,191 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:15:49,025 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:15:51,026 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:15:52,515 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:15:52,516 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:15:55,029 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:15:55,029 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:15:55,030 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:15:58,072 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:16:03,073 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:16:08,074 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:16:08,324 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:16:08,325 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:16:08,325 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:16:08,326 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:16:09,033 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:16:10,029 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:16:10,029 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:16:10,030 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:16:11,034 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:16:13,156 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:16:18,157 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:16:22,516 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:16:23,517 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:16:25,029 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:16:25,029 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:16:25,030 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:16:28,113 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:16:28,115 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:16:28,115 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:16:28,115 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:16:28,903 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:16:29,040 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:16:31,041 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:16:33,904 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:16:38,905 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:16:40,029 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:16:40,029 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:16:40,030 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:16:44,063 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:16:49,064 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:16:49,759 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:16:49,760 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:16:49,760 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:16:49,761 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:16:50,049 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:16:52,517 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:16:53,050 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:16:54,518 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:16:55,029 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:16:55,030 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:16:55,030 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:17:00,107 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:17:05,108 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:17:09,918 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:17:09,919 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:17:09,920 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:17:09,920 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:17:10,029 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:17:10,030 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:17:10,030 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:17:10,056 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:17:10,151 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:17:13,058 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:17:15,650 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:17:20,651 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:17:22,518 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:17:25,029 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:17:25,030 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:17:25,031 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:17:26,156 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:17:28,550 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:17:28,551 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:17:28,551 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:17:28,552 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:17:29,063 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:17:31,064 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:17:31,274 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:17:36,275 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:17:40,033 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:17:40,034 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:17:40,034 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:17:42,145 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:17:47,146 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:17:49,771 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:17:49,773 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:17:49,773 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:17:49,773 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:17:50,072 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:17:52,519 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:17:52,520 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:17:53,073 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:17:55,034 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:17:55,034 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:17:55,074 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:17:58,122 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:18:03,123 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:18:08,124 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:18:10,032 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:18:10,033 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:18:10,073 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:18:10,453 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:18:10,454 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:18:10,454 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:18:10,455 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:18:11,079 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:18:13,081 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:18:13,188 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:18:18,189 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:18:22,520 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:18:23,521 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:18:25,031 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:18:25,033 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:18:25,033 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:18:29,104 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:18:32,854 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:18:32,855 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:18:32,856 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:18:32,856 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:18:33,089 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:18:34,814 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:18:35,090 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:18:39,815 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:18:40,032 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:18:40,033 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:18:40,033 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:18:45,091 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:18:50,092 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:18:52,521 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:18:53,121 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:18:53,122 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:18:53,123 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:18:53,123 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:18:54,097 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:18:55,032 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:18:55,033 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:18:55,033 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:18:55,097 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:18:56,082 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:19:01,083 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:19:06,084 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:19:10,034 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:19:10,036 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:19:10,036 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:19:12,063 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:19:15,185 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:19:15,186 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:19:15,186 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:19:15,187 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:19:16,105 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:19:17,105 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:19:17,967 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:19:22,521 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:19:23,523 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:19:25,032 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:19:25,033 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:19:25,034 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:19:29,068 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:19:34,068 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:19:34,450 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:19:34,451 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:19:34,452 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:19:34,452 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:19:35,112 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:19:37,113 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:19:39,205 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:19:40,032 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:19:40,033 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:19:40,034 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:19:45,161 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:19:50,161 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:19:52,523 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:19:55,033 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:19:55,034 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:19:55,034 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:19:55,376 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:19:55,378 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:19:55,378 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:19:55,378 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:19:55,379 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:19:56,119 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:19:57,119 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:20:01,126 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:20:06,127 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:20:10,033 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:20:10,033 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:20:10,034 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:20:11,152 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:20:15,937 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:20:15,938 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:20:15,939 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:20:15,939 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:20:16,126 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:20:16,716 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:20:19,127 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:20:21,716 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:20:22,523 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:20:25,033 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:20:25,033 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:20:25,034 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:20:27,154 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:20:32,155 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:20:35,859 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:20:35,860 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:20:35,860 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:20:35,861 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:20:36,134 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:20:37,619 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:20:39,135 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:20:40,033 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:20:40,033 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:20:40,034 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:20:43,161 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:20:48,162 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:20:52,528 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:20:53,529 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:20:55,033 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:20:55,034 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:20:55,034 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:20:56,689 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:20:56,690 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:20:56,690 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:20:56,691 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:20:57,141 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:20:58,537 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:20:59,142 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:21:03,537 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:21:08,538 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:21:10,034 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:21:10,036 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:21:10,037 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:21:14,114 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:21:16,643 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:21:16,645 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:21:16,645 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:21:16,645 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:21:17,148 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:21:19,149 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:21:19,411 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:21:22,525 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:21:24,526 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:21:25,033 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:21:25,034 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:21:25,035 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:21:30,123 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:21:34,403 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:21:34,404 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:21:34,404 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:21:34,406 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:21:35,152 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:21:35,155 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:21:37,156 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:21:40,034 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:21:40,034 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:21:40,035 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:21:40,157 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:21:45,157 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:21:50,158 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:21:51,923 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:21:51,925 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:21:51,925 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:21:51,926 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:21:52,162 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:21:52,526 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:21:55,034 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:21:55,034 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:21:55,035 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:21:55,163 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:21:56,094 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:22:01,095 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:22:06,096 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:22:10,034 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:22:10,036 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:22:10,037 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:22:11,156 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:22:14,792 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:22:14,794 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:22:14,794 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:22:14,796 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:22:15,170 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:22:16,609 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:22:17,171 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:22:21,610 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:22:22,527 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:22:25,034 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:22:25,035 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:22:25,036 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:22:27,095 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:22:32,096 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:22:37,097 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:22:37,925 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:22:37,926 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:22:37,927 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:22:37,927 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:22:38,179 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:22:40,034 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:22:40,035 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:22:40,036 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:22:41,180 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:22:42,157 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:22:47,158 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:22:52,158 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:22:52,528 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:22:55,035 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:22:55,035 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:22:55,036 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:22:57,827 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:22:57,828 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:22:57,828 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:22:57,828 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:22:57,829 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:22:58,187 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:22:59,187 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:23:03,594 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:23:08,595 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:23:10,035 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:23:10,035 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:23:10,036 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:23:14,104 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:23:19,022 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:23:19,023 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:23:19,024 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:23:19,024 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:23:19,194 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:23:19,913 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:23:21,195 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:23:22,529 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:23:25,035 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:23:25,035 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:23:25,036 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:23:25,086 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:23:30,087 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:23:35,088 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:23:39,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:23:39,439 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:23:39,440 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:23:39,440 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:23:40,147 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:23:40,148 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:23:40,152 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:23:40,201 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:23:40,235 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:23:41,202 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:23:45,236 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:23:50,237 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:23:52,530 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:23:55,145 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:23:55,145 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:23:55,185 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:23:55,241 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:23:59,600 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:23:59,601 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:23:59,601 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:23:59,602 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:24:00,210 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:24:00,306 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:24:01,211 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:24:05,308 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:24:10,150 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:24:10,150 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:24:10,151 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:24:11,255 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:24:16,256 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:24:21,257 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:24:21,300 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:24:21,301 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:24:21,301 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:24:21,302 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:24:22,219 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:24:22,532 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:24:23,219 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:24:25,147 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:24:25,148 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:24:25,149 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:24:27,212 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:24:32,213 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:24:37,214 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:24:40,148 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:24:40,148 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:24:40,148 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:24:41,883 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:24:41,884 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:24:41,884 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:24:41,885 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:24:42,226 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:24:42,607 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:24:43,227 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:24:47,608 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:24:52,533 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:24:53,533 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:24:55,148 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:24:55,148 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:24:55,149 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:24:59,214 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:25:03,612 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:25:03,613 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:25:03,614 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:25:03,614 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:25:04,234 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:25:04,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:25:05,234 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:25:09,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:25:10,149 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:25:10,149 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:25:10,152 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:25:15,236 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:25:20,237 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:25:22,533 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:25:25,148 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:25:25,149 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:25:25,149 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:25:25,270 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:25:26,797 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:25:26,798 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:25:26,798 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:25:26,799 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:25:27,242 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:25:29,242 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:25:30,689 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:25:35,690 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:25:40,148 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:25:40,148 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:25:40,149 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:25:41,267 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:25:46,268 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:25:47,698 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:25:47,700 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:25:47,700 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:25:47,700 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:25:48,250 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:25:49,251 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:25:51,409 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:25:52,535 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:25:55,148 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:25:55,148 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:25:55,149 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:25:57,227 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:26:02,228 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:26:07,229 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:26:10,149 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:26:10,149 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:26:10,189 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:26:10,515 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:26:10,517 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:26:10,517 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:26:10,517 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:26:11,259 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:26:12,351 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:26:13,260 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:26:17,352 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:26:22,352 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:26:22,536 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:26:25,148 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:26:25,148 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:26:25,149 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:26:28,236 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:26:29,834 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:26:29,835 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:26:29,836 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:26:29,836 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:26:30,267 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:26:31,267 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:26:33,632 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:26:38,633 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:26:40,148 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:26:40,148 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:26:40,149 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:26:44,209 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:26:49,210 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:26:52,162 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:26:52,163 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:26:52,163 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:26:52,164 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:26:52,275 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:26:52,537 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:26:53,275 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:26:54,932 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:26:55,156 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:26:55,159 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:26:55,159 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:27:00,207 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:27:05,208 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:27:10,150 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:27:10,151 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:27:10,151 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:27:11,197 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:27:12,602 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:27:12,604 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:27:12,604 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:27:12,604 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:27:13,282 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:27:15,283 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:27:16,326 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:27:21,326 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:27:22,538 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:27:25,150 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:27:25,151 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:27:25,151 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:27:27,234 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:27:32,234 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:27:35,130 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:27:35,131 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:27:35,131 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:27:35,132 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:27:35,290 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:27:37,291 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:27:37,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:27:40,151 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:27:40,151 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:27:40,152 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:27:43,257 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:27:48,258 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:27:52,538 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:27:53,539 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:27:55,154 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:27:55,159 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:27:55,160 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:27:55,200 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:27:55,218 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:27:55,218 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:27:55,218 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:27:55,298 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:27:57,299 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:27:59,294 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:28:04,295 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:28:09,296 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:28:10,151 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:28:10,151 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:28:10,152 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:28:15,209 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:28:20,210 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:28:20,348 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:28:20,349 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:28:20,350 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:28:20,350 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:28:21,309 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:28:22,540 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:28:23,309 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:28:25,151 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:28:25,151 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:28:25,152 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:28:25,232 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:28:30,233 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:28:35,234 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:28:40,151 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:28:40,152 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:28:40,153 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:28:40,235 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:28:42,598 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:28:42,599 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:28:42,600 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:28:42,600 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:28:43,318 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:28:45,318 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:28:45,500 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:28:50,502 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:28:52,545 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:28:55,151 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:28:55,153 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:28:55,153 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:28:56,221 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:29:01,222 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:29:05,520 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:29:05,522 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:29:05,522 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:29:05,522 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:29:06,258 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:29:06,326 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:29:07,326 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:29:10,151 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:29:10,152 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:29:10,152 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:29:11,264 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:29:16,265 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:29:21,266 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:29:22,542 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:29:25,151 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:29:25,152 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:29:25,152 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:29:27,255 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:29:28,538 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:29:28,539 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:29:28,539 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:29:28,541 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:29:29,334 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:29:31,334 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:29:32,280 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:29:37,281 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:29:40,151 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:29:40,152 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:29:40,152 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:29:43,271 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:29:48,272 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:29:48,590 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:29:48,592 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:29:48,592 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:29:48,593 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:29:49,341 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:29:52,342 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:29:52,543 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:29:53,544 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:29:55,152 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:29:55,153 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:29:55,153 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:29:59,259 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:30:04,260 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:30:09,261 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:30:09,625 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:30:09,626 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:30:09,627 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:30:09,629 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:30:10,321 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:30:10,322 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:30:10,325 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:30:10,347 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:30:12,348 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:30:14,426 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:30:19,427 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:30:22,544 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:30:24,545 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:30:25,319 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:30:25,320 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:30:25,320 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:30:29,490 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:30:29,491 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:30:29,491 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:30:29,491 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:30:30,356 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:30:30,391 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:30:32,357 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:30:35,392 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:30:40,319 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:30:40,320 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:30:40,359 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:30:40,434 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:30:45,435 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:30:50,435 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:30:51,398 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:30:51,399 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:30:51,399 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:30:51,401 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:30:52,365 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:30:52,545 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:30:54,365 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:30:55,319 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:30:55,319 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:30:55,359 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:30:56,432 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:31:01,433 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:31:06,434 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:31:10,319 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:31:10,319 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:31:10,360 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:31:11,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:31:12,934 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:31:12,935 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:31:12,935 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:31:12,936 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:31:13,372 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:31:16,374 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:31:16,667 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:31:21,668 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:31:22,546 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:31:25,319 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:31:25,319 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:31:25,359 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:31:27,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:31:32,449 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:31:34,118 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:31:34,119 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:31:34,119 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:31:34,120 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:31:34,382 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:31:36,383 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:31:37,855 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:31:40,319 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:31:40,319 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:31:40,360 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:31:43,385 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:31:48,386 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:31:52,546 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:31:53,547 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:31:55,190 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:31:55,191 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:31:55,191 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:31:55,192 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:31:55,319 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:31:55,319 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:31:55,359 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:31:55,390 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:31:58,391 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:31:58,927 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:32:03,928 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:32:08,929 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:32:10,320 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:32:10,320 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:32:10,361 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:32:14,424 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:32:15,239 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:32:15,241 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:32:15,241 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:32:15,241 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:32:15,397 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:32:18,399 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:32:20,023 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:32:22,547 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:32:25,318 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:32:25,319 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:32:25,320 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:32:25,359 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:32:30,432 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:32:35,002 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:32:35,003 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:32:35,003 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:32:35,004 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:32:35,405 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:32:35,745 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:32:38,406 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:32:40,319 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:32:40,320 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:32:40,360 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:32:41,450 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:32:46,451 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:32:51,452 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:32:52,549 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:32:55,320 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:32:55,320 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:32:55,361 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:32:57,450 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:32:58,963 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:32:58,965 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:32:58,965 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:32:58,965 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:32:59,414 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:33:02,415 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:33:02,912 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:33:07,913 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:33:10,320 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:33:10,320 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:33:10,361 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:33:13,423 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:33:18,424 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:33:19,914 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:33:19,915 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:33:19,916 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:33:19,916 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:33:20,422 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:33:22,423 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:33:22,549 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:33:23,550 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:33:25,320 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:33:25,320 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:33:25,360 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:33:29,406 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:33:34,407 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:33:39,408 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:33:39,969 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:33:39,971 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:33:39,971 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:33:39,971 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:33:40,427 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:33:40,428 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:33:40,428 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:33:40,434 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:33:42,435 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:33:44,723 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:33:49,724 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:33:52,550 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:33:55,381 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:33:55,381 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:33:55,386 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:33:55,503 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:34:00,504 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:34:02,453 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:34:02,454 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:34:02,454 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:34:02,455 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:34:03,442 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:34:04,443 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:34:06,181 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:34:10,381 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:34:10,381 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:34:10,386 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:34:11,474 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:34:16,475 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:34:21,476 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:34:22,551 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:34:22,576 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:34:22,577 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:34:22,578 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:34:22,578 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:34:23,450 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:34:24,451 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:34:25,381 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:34:25,381 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:34:25,387 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:34:27,461 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:34:32,462 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:34:37,463 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:34:40,381 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:34:40,381 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:34:40,420 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:34:42,514 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:34:42,515 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:34:42,515 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:34:42,516 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:34:42,516 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:34:43,457 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:34:44,458 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:34:48,420 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:34:52,552 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:34:53,553 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:34:55,381 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:34:55,382 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:34:55,387 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:34:59,507 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:35:04,508 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:35:04,987 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:35:04,988 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:35:04,989 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:35:04,989 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:35:05,464 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:35:08,466 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:35:09,757 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:35:10,381 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:35:10,382 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:35:10,387 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:35:15,478 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:35:20,479 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:35:22,553 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:35:25,264 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:35:25,265 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:35:25,266 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:35:25,266 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:35:25,381 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:35:25,382 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:35:25,387 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:35:25,472 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:35:25,498 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:35:28,474 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:35:31,012 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:35:36,014 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:35:40,382 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:35:40,382 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:35:40,388 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:35:41,514 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:35:45,234 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:35:45,235 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:35:45,236 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:35:45,236 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:35:45,479 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:35:47,268 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:35:48,481 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:35:52,269 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:35:52,554 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:35:55,382 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:35:55,382 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:35:55,387 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:35:57,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:36:02,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:36:06,365 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:36:06,366 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:36:06,366 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:36:06,367 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:36:06,488 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:36:08,110 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:36:08,489 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:36:10,382 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:36:10,382 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:36:10,387 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:36:13,480 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:36:18,481 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:36:22,555 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:36:23,555 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:36:25,382 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:36:25,383 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:36:25,423 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:36:25,423 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:36:25,435 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:36:25,435 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:36:25,435 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:36:25,495 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:36:28,496 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:36:29,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:36:34,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:36:39,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:36:40,382 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:36:40,383 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:36:40,391 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:36:44,506 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:36:45,336 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:36:45,338 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:36:45,338 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:36:45,338 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:36:45,502 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:36:48,504 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:36:50,056 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:36:52,555 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:36:55,383 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:36:55,383 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:36:55,391 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:36:55,503 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:37:00,504 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:37:05,505 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:37:08,748 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:37:08,749 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:37:08,750 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:37:08,750 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:37:09,512 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:37:10,383 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:37:10,383 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:37:10,392 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:37:10,512 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:37:11,403 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:37:16,405 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:37:21,406 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:37:22,556 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:37:25,383 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:37:25,383 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:37:25,424 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:37:26,490 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:37:28,486 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:37:28,487 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:37:28,487 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:37:28,487 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:37:28,519 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:37:30,520 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:37:32,453 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:37:37,453 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:37:40,384 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:37:40,384 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:37:40,400 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:37:43,449 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:37:48,410 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:37:48,411 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:37:48,411 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:37:48,412 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:37:48,527 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:37:49,134 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:37:50,528 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:37:52,557 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:37:54,558 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:37:55,384 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:37:55,384 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:37:55,397 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:38:00,477 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:38:05,477 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:38:10,038 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:38:10,039 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:38:10,039 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:38:10,040 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:38:10,476 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:38:10,477 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:38:10,487 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:38:10,521 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:38:10,536 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:38:12,537 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:38:15,774 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:38:20,775 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:38:22,558 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:38:25,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:38:25,442 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:38:25,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:38:26,531 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:38:31,532 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:38:33,516 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:38:33,517 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:38:33,518 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:38:33,518 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:38:33,544 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:38:36,545 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:38:37,264 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:38:40,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:38:40,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:38:40,444 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:38:42,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:38:47,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:38:51,015 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:38:51,016 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:38:51,017 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:38:51,017 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:38:51,551 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:38:52,559 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:38:53,560 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:38:54,553 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:38:55,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:38:55,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:38:55,444 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:38:59,556 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:39:04,556 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:39:09,557 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:39:10,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:39:10,443 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:39:10,483 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:39:12,746 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:39:12,748 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:39:12,748 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:39:12,748 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:39:13,561 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:39:14,561 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:39:15,537 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:39:20,538 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:39:22,560 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:39:25,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:39:25,444 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:39:25,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:39:25,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:39:30,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:39:34,805 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:39:34,806 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:39:34,806 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:39:34,807 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:39:35,569 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:39:36,552 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:39:36,569 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:39:40,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:39:40,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:39:40,443 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:39:41,556 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:39:46,558 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:39:51,559 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:39:52,561 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:39:55,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:39:55,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:39:55,443 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:39:57,217 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:39:57,218 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:39:57,219 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:39:57,219 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:39:57,219 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:39:57,577 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:39:58,577 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:40:02,958 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:40:07,959 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:40:10,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:40:10,443 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:40:10,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:40:13,551 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:40:17,906 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:40:17,907 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:40:17,907 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:40:17,908 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:40:18,585 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:40:18,677 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:40:20,586 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:40:22,562 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:40:24,563 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:40:25,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:40:25,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:40:25,444 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:40:30,520 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:40:35,521 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:40:40,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:40:40,443 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:40:40,484 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:40:41,019 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:40:41,020 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:40:41,021 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:40:41,021 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:40:41,021 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:40:41,595 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:40:42,596 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:40:46,810 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:40:51,811 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:40:52,563 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:40:55,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:40:55,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:40:55,445 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:40:57,483 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:41:02,149 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:41:02,151 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:41:02,151 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:41:02,151 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:41:02,603 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:41:02,898 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:41:04,604 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:41:07,899 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:41:10,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:41:10,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:41:10,445 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:41:13,521 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:41:18,522 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:41:22,564 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:41:23,323 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:41:23,325 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:41:23,325 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:41:23,325 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:41:23,612 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:41:24,072 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:41:25,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:41:25,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:41:25,444 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:41:26,613 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:41:29,509 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:41:34,510 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:41:39,511 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:41:40,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:41:40,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:41:40,444 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:41:44,392 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:41:44,393 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:41:44,394 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:41:44,394 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:41:44,619 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:41:45,165 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:41:46,620 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:41:50,166 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:41:52,565 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:41:55,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:41:55,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:41:55,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:41:55,445 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:42:00,597 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:42:03,849 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:42:03,850 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:42:03,851 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:42:03,851 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:42:04,627 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:42:05,650 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:42:06,628 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:42:10,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:42:10,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:42:10,445 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:42:11,542 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:42:16,543 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:42:21,544 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:42:22,566 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:42:23,014 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:42:23,015 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:42:23,015 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:42:23,016 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:42:23,634 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:42:24,635 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:42:25,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:42:25,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:42:25,444 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:42:26,576 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:42:31,575 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:42:36,576 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:42:40,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:42:40,444 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:42:40,484 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:42:42,511 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:42:42,554 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:42:42,555 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:42:42,555 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:42:42,556 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:42:42,642 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:42:44,642 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:42:48,296 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:42:52,566 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:42:53,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:42:55,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:42:55,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:42:55,445 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:42:58,575 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:43:01,981 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:43:01,982 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:43:01,982 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:43:01,983 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:43:02,649 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:43:03,739 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:43:04,650 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:43:08,740 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:43:10,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:43:10,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:43:10,445 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:43:14,578 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:43:19,579 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:43:21,670 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:43:21,672 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:43:21,672 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:43:21,672 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:43:22,568 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:43:22,657 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:43:24,658 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:43:25,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:43:25,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:43:25,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:43:25,446 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:43:30,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:43:35,571 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:43:40,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:43:40,445 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:43:40,484 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:43:41,484 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:43:42,622 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:43:42,623 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:43:42,623 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:43:42,624 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:43:42,666 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:43:44,666 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:43:47,419 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:43:52,420 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:43:52,569 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:43:55,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:43:55,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:43:55,445 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:43:57,504 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:44:02,505 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:44:04,549 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:44:04,551 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:44:04,551 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:44:04,552 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:44:04,674 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:44:06,675 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:44:08,370 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:44:10,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:44:10,446 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:44:10,486 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:44:13,555 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:44:18,556 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:44:22,569 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:44:23,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:44:24,590 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:44:24,591 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:44:24,591 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:44:24,592 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:44:24,682 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:44:25,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:44:25,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:44:25,448 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:44:26,683 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:44:29,493 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:44:34,495 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:44:39,496 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:44:40,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:44:40,446 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:44:40,486 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:44:44,003 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:44:44,005 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:44:44,005 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:44:44,005 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:44:44,690 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:44:44,743 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:44:46,690 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:44:49,744 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:44:52,571 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:44:55,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:44:55,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:44:55,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:44:55,446 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:45:00,492 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:45:02,693 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:45:02,695 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:45:02,695 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:45:02,695 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:45:02,697 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:45:04,698 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:45:06,404 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:45:10,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:45:10,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:45:10,446 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:45:11,529 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:45:16,530 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:45:21,530 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:45:22,576 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:45:22,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:45:22,736 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:45:22,736 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:45:22,736 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:45:23,705 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:45:24,705 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:45:25,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:45:25,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:45:25,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:45:26,536 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:45:31,537 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:45:36,538 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:45:40,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:45:40,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:45:40,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:45:42,474 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:45:44,165 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:45:44,167 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:45:44,167 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:45:44,167 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:45:44,713 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:45:46,714 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:45:47,882 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:45:52,573 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:45:53,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:45:55,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:45:55,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:45:55,448 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:45:59,496 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:46:03,932 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:46:03,933 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:46:03,933 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:46:03,935 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:46:04,720 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:46:04,936 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:46:06,721 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:46:09,988 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:46:10,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:46:10,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:46:10,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:46:15,514 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:46:20,515 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:46:22,574 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:46:25,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:46:25,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:46:25,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:46:26,433 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:46:26,435 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:46:26,435 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:46:26,435 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:46:26,436 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:46:26,729 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:46:28,730 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:46:32,161 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:46:37,162 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:46:40,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:46:40,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:46:40,448 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:46:42,520 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:46:46,076 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:46:46,077 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:46:46,078 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:46:46,080 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:46:46,737 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:46:47,826 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:46:48,738 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:46:52,575 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:46:53,576 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:46:55,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:46:55,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:46:55,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:46:59,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:47:04,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:47:07,960 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:47:07,961 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:47:07,961 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:47:07,962 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:47:08,746 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:47:09,703 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:47:10,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:47:10,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:47:10,448 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:47:10,747 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:47:15,503 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:47:20,504 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:47:22,576 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:47:25,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:47:25,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:47:25,448 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:47:25,569 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:47:29,516 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:47:29,517 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:47:29,518 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:47:29,518 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:47:29,755 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:47:31,230 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:47:32,756 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:47:36,231 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:47:40,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:47:40,448 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:47:40,488 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:47:41,531 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:47:46,532 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:47:48,474 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:47:48,476 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:47:48,476 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:47:48,476 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:47:48,762 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:47:50,763 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:47:52,362 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:47:52,577 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:47:55,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:47:55,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:47:55,449 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:47:57,530 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:48:02,531 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:48:07,532 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:48:09,230 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:48:09,231 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:48:09,232 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:48:09,232 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:48:09,771 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:48:10,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:48:10,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:48:10,448 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:48:10,771 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:48:13,530 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:48:18,531 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:48:22,578 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:48:23,579 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:48:25,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:48:25,449 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:48:25,449 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:48:29,577 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:48:29,618 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:48:29,619 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:48:29,619 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:48:29,620 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:48:29,778 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:48:32,780 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:48:35,351 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:48:40,352 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:48:40,453 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:48:40,454 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:48:40,458 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:48:45,598 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:48:50,599 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:48:52,204 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:48:52,205 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:48:52,206 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:48:52,206 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:48:52,579 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:48:52,787 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:48:54,788 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:48:55,450 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:48:55,451 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:48:55,452 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:48:56,475 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:49:01,476 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:49:06,477 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:49:10,450 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:49:10,451 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:49:10,451 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:49:11,492 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:49:13,726 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:49:13,728 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:49:13,728 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:49:13,728 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:49:13,795 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:49:16,501 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:49:16,797 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:49:21,504 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:49:22,580 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:49:25,451 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:49:25,452 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:49:25,452 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:49:26,506 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:49:31,507 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:49:34,119 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:49:34,120 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:49:34,121 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:49:34,121 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:49:34,804 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:49:36,805 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:49:36,834 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:49:40,452 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:49:40,452 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:49:40,453 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:49:42,520 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:49:47,521 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:49:52,522 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:49:52,581 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:49:54,846 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:49:54,847 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:49:54,847 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:49:54,848 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:49:55,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:49:55,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:49:55,568 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:49:55,813 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:49:56,814 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:49:57,666 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:50:02,667 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:50:07,668 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:50:10,562 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:50:10,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:50:10,564 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:50:13,614 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:50:13,661 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:50:13,662 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:50:13,662 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:50:13,663 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:50:13,820 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:50:16,821 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:50:19,397 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:50:22,582 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:50:24,582 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:50:25,562 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:50:25,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:50:25,565 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:50:29,626 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:50:33,684 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:50:33,686 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:50:33,686 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:50:33,686 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:50:33,828 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:50:35,394 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:50:36,829 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:50:40,395 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:50:40,562 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:50:40,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:50:40,565 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:50:45,621 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:50:50,622 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:50:51,885 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:50:51,886 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:50:51,887 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:50:51,887 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:50:52,583 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:50:52,834 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:50:54,836 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:50:55,563 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:50:55,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:50:55,565 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:50:55,664 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:51:00,665 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:51:05,666 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:51:10,563 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:51:10,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:51:10,565 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:51:10,818 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:51:10,819 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:51:10,819 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:51:10,819 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:51:10,820 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:51:10,841 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:51:12,842 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:51:16,756 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:51:21,757 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:51:22,583 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:51:25,563 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:51:25,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:51:25,565 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:51:27,650 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:51:31,423 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:51:31,425 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:51:31,425 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:51:31,425 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:51:31,850 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:51:32,850 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:51:33,150 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:51:38,151 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:51:40,563 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:51:40,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:51:40,566 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:51:43,643 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:51:48,644 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:51:49,221 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:51:49,222 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:51:49,223 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:51:49,223 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:51:49,858 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:51:50,858 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:51:52,585 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:51:54,586 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:51:55,563 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:51:55,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:51:55,565 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:51:59,626 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:52:04,627 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:52:09,628 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:52:10,563 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:52:10,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:52:10,566 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:52:11,319 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:52:11,321 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:52:11,321 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:52:11,321 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:52:11,866 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:52:12,866 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:52:15,041 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:52:20,042 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:52:22,586 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:52:25,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:52:25,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:52:25,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:52:25,566 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:52:30,661 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:52:31,485 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:52:31,486 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:52:31,486 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:52:31,487 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:52:31,873 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:52:33,874 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:52:36,220 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:52:40,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:52:40,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:52:40,566 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:52:41,648 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:52:46,649 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:52:51,414 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:52:51,415 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:52:51,416 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:52:51,416 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:52:51,882 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:52:52,156 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:52:52,587 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:52:53,883 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:52:55,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:52:55,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:52:55,566 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:52:57,684 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:53:02,685 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:53:07,685 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:53:10,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:53:10,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:53:10,566 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:53:12,084 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:53:12,085 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:53:12,086 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:53:12,086 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:53:12,808 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:53:12,890 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:53:15,892 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:53:17,809 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:53:22,592 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:53:23,593 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:53:25,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:53:25,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:53:25,567 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:53:28,675 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:53:32,244 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:53:32,245 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:53:32,245 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:53:32,246 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:53:32,898 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:53:33,973 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:53:35,899 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:53:38,974 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:53:40,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:53:40,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:53:40,567 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:53:44,670 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:53:49,671 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:53:51,310 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:53:51,311 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:53:51,312 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:53:51,312 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:53:51,906 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:53:52,589 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:53:53,907 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:53:55,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:53:55,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:53:55,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:53:55,569 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:54:00,645 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:54:05,646 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:54:10,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:54:10,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:54:10,568 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:54:10,690 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:54:13,815 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:54:13,816 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:54:13,817 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:54:13,817 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:54:13,914 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:54:15,750 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:54:17,916 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:54:20,751 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:54:22,589 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:54:25,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:54:25,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:54:25,568 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:54:26,614 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:54:31,614 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:54:34,022 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:54:34,023 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:54:34,024 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:54:34,026 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:54:34,922 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:54:36,789 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:54:37,923 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:54:40,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:54:40,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:54:40,567 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:54:42,663 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:54:47,664 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:54:52,590 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:54:53,591 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:54:55,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:54:55,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:54:55,568 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:54:56,888 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:54:56,889 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:54:56,889 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:54:56,889 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:54:56,930 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:54:58,626 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:54:59,931 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:55:03,627 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:55:08,628 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:55:10,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:55:10,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:55:10,568 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:55:13,647 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:55:17,417 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:55:17,418 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:55:17,419 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:55:17,421 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:55:17,938 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:55:19,149 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:55:19,939 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:55:22,591 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:55:24,592 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:55:25,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:55:25,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:55:25,568 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:55:29,620 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:55:34,621 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:55:39,417 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:55:39,419 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:55:39,419 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:55:39,420 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:55:39,946 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:55:40,205 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:55:40,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:55:40,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:55:40,568 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:55:41,947 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:55:45,696 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:55:50,697 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:55:52,592 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:55:55,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:55:55,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:55:55,568 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:55:56,668 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:56:00,890 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:56:00,891 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:56:00,892 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:56:00,892 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:56:00,955 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:56:02,631 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:56:03,956 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:56:07,632 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:56:10,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:56:10,569 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:56:10,569 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:56:12,669 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:56:17,670 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:56:22,593 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:56:23,594 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:56:25,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:56:25,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:56:25,570 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:56:26,429 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:56:26,430 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:56:26,430 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:56:26,431 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:56:26,966 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:56:29,183 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:56:29,967 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:56:34,184 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:56:39,185 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:56:40,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:56:40,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:56:40,571 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:56:44,668 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:56:48,490 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:56:48,491 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:56:48,491 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:56:48,492 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:56:48,975 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:56:50,259 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:56:51,977 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:56:52,594 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:56:55,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:56:55,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:56:55,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:56:55,570 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:57:00,684 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:57:05,685 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:57:09,872 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:57:09,873 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:57:09,873 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:57:09,874 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:57:09,985 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:57:10,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:57:10,736 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:57:10,738 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:57:10,867 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:57:13,986 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:57:15,868 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:57:20,869 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:57:22,595 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:57:25,730 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:57:25,733 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:57:25,734 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:57:26,817 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:57:31,818 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:57:33,494 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:57:33,495 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:57:33,495 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:57:33,496 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:57:33,995 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:57:35,996 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:57:37,268 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:57:40,730 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:57:40,733 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:57:40,733 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:57:42,810 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:57:47,811 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:57:52,596 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:57:53,597 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:57:55,730 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:57:55,733 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:57:55,733 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:57:58,165 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:57:58,166 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:57:58,166 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:57:58,166 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:57:58,892 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:57:59,005 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:58:02,007 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:58:03,892 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:58:08,893 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:58:10,730 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:58:10,733 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:58:10,733 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:58:14,838 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:58:19,839 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:58:21,282 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:58:21,283 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:58:21,283 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:58:21,284 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:58:22,014 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:58:22,597 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:58:24,015 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:58:25,598 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:58:25,730 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:58:25,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:58:25,734 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:58:30,882 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:58:35,883 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:58:40,731 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:58:40,733 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:58:40,734 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:58:41,811 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:58:43,136 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:58:43,137 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:58:43,138 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:58:43,138 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:58:44,022 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:58:46,023 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:58:46,930 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:58:51,931 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:58:52,598 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:58:55,731 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:58:55,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:58:55,734 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:58:57,827 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:59:02,828 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:59:06,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:59:06,441 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:59:06,442 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:59:06,442 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:59:07,031 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:59:08,171 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:59:10,032 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:59:10,731 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:59:10,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:59:10,734 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:59:13,782 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:59:18,783 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:59:22,599 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:59:24,599 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:59:25,731 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:59:25,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:59:25,734 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:59:26,179 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:59:26,180 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:59:26,180 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:59:26,180 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:59:27,039 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:59:28,040 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:59:29,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:59:34,944 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:59:39,945 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:59:40,731 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:59:40,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:59:40,734 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:59:45,661 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 13:59:45,662 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 13:59:45,663 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 13:59:45,663 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:59:45,664 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 13:59:46,046 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 13:59:48,047 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 13:59:51,485 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 13:59:52,599 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 13:59:55,732 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 13:59:55,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 13:59:55,735 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 13:59:56,868 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:00:01,869 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:00:04,996 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:00:04,998 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:00:04,998 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:00:04,998 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:00:05,053 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:00:07,715 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:00:08,054 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:00:10,732 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:00:10,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:00:10,734 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:00:12,851 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:00:17,852 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:00:22,600 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:00:23,601 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:00:25,732 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:00:25,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:00:25,735 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:00:28,808 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:00:28,863 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:00:28,865 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:00:28,865 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:00:28,865 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:00:29,062 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:00:32,063 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:00:34,602 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:00:39,602 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:00:40,732 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:00:40,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:00:40,735 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:00:44,776 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:00:49,413 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:00:49,415 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:00:49,415 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:00:49,415 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:00:50,070 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:00:50,218 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:00:52,071 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:00:52,601 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:00:55,602 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:00:55,732 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:00:55,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:00:55,735 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:01:00,836 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:01:05,837 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:01:10,732 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:01:10,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:01:10,735 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:01:11,776 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:01:12,160 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:01:12,162 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:01:12,162 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:01:12,162 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:01:13,080 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:01:14,080 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:01:16,891 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:01:21,892 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:01:22,602 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:01:25,732 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:01:25,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:01:25,735 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:01:27,795 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:01:30,873 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:01:30,875 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:01:30,875 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:01:30,875 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:01:31,087 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:01:33,615 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:01:34,088 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:01:38,616 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:01:40,733 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:01:40,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:01:40,735 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:01:43,828 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:01:48,829 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:01:50,392 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:01:50,397 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:01:50,397 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:01:50,398 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:01:51,094 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:01:52,603 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:01:54,096 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:01:54,604 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:01:55,733 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:01:55,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:01:55,736 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:01:59,820 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:02:04,821 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:02:08,891 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:02:08,892 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:02:08,892 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:02:08,893 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:02:09,101 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:02:10,621 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:02:10,733 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:02:10,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:02:10,735 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:02:12,102 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:02:15,804 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:02:20,806 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:02:22,605 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:02:25,733 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:02:25,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:02:25,735 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:02:26,793 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:02:29,597 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:02:29,598 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:02:29,599 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:02:29,599 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:02:30,109 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:02:32,110 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:02:32,318 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:02:37,319 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:02:40,733 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:02:40,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:02:40,736 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:02:42,862 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:02:47,863 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:02:51,119 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:02:51,120 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:02:51,120 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:02:51,121 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:02:52,118 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:02:52,606 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:02:53,607 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:02:54,119 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:02:55,733 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:02:55,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:02:55,736 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:02:58,841 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:03:03,842 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:03:08,843 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:03:10,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:03:10,736 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:03:10,736 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:03:12,097 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:03:12,098 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:03:12,098 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:03:12,099 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:03:12,126 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:03:14,127 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:03:14,824 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:03:19,825 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:03:22,606 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:03:25,607 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:03:25,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:03:25,736 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:03:25,736 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:03:29,784 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:03:29,786 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:03:29,786 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:03:29,787 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:03:30,133 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:03:31,511 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:03:32,134 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:03:36,512 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:03:40,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:03:40,737 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:03:40,737 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:03:41,794 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:03:46,795 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:03:49,423 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:03:49,424 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:03:49,424 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:03:49,424 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:03:50,141 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:03:52,136 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:03:52,142 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:03:52,607 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:03:55,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:03:55,741 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:03:55,742 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:03:57,840 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:04:02,841 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:04:07,842 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:04:08,472 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:04:08,473 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:04:08,473 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:04:08,474 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:04:09,148 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:04:10,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:04:10,736 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:04:10,737 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:04:12,149 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:04:12,845 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:04:17,846 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:04:22,608 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:04:23,609 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:04:25,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:04:25,736 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:04:25,737 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:04:28,674 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:04:28,675 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:04:28,675 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:04:28,676 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:04:28,676 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:04:29,156 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:04:32,157 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:04:34,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:04:39,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:04:40,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:04:40,736 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:04:40,737 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:04:44,795 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:04:49,796 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:04:50,366 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:04:50,367 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:04:50,367 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:04:50,368 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:04:51,164 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:04:52,164 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:04:52,609 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:04:55,610 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:04:55,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:04:55,737 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:04:55,737 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:05:00,783 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:05:05,784 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:05:10,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:05:10,737 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:05:10,737 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:05:11,297 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:05:11,298 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:05:11,298 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:05:11,299 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:05:11,299 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:05:12,171 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:05:14,173 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:05:17,074 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:05:22,074 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:05:22,610 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:05:25,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:05:25,737 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:05:25,737 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:05:27,829 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:05:31,392 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:05:31,393 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:05:31,393 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:05:31,394 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:05:32,180 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:05:33,115 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:05:34,181 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:05:38,116 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:05:40,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:05:40,737 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:05:40,737 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:05:43,808 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:05:48,809 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:05:52,611 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:05:53,606 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:05:53,607 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:05:53,608 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:05:53,608 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:05:54,188 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:05:54,579 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:05:55,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:05:55,737 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:05:55,737 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:05:56,189 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:05:59,848 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:06:04,849 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:06:09,850 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:06:10,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:06:10,737 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:06:10,738 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:06:14,852 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:06:15,269 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:06:15,270 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:06:15,271 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:06:15,271 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:06:16,198 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:06:18,199 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:06:20,016 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:06:22,612 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:06:25,613 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:06:25,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:06:25,737 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:06:25,738 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:06:30,788 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:06:35,789 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:06:36,463 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:06:36,464 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:06:36,464 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:06:36,464 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:06:37,206 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:06:38,207 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:06:40,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:06:40,738 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:06:40,738 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:06:40,796 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:06:45,797 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:06:50,798 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:06:52,613 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:06:55,736 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:06:55,739 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:06:55,739 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:06:55,853 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:06:58,530 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:06:58,531 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:06:58,531 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:06:58,532 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:06:59,215 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:07:00,216 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:07:01,281 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:07:06,282 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:07:10,736 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:07:10,738 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:07:10,739 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:07:11,867 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:07:16,867 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:07:18,751 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:07:18,752 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:07:18,753 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:07:18,753 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:07:19,223 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:07:20,223 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:07:22,483 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:07:22,614 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:07:25,736 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:07:25,738 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:07:25,739 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:07:27,867 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:07:32,868 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:07:37,869 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:07:40,428 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:07:40,429 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:07:40,429 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:07:40,429 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:07:40,888 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:07:40,888 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:07:40,889 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:07:41,230 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:07:42,231 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:07:43,178 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:07:48,179 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:07:52,615 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:07:53,616 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:07:55,844 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:07:55,851 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:07:55,852 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:07:58,930 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:08:03,932 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:08:05,023 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:08:05,024 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:08:05,024 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:08:05,025 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:08:05,240 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:08:08,241 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:08:09,803 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:08:10,843 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:08:10,850 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:08:10,850 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:08:14,931 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:08:19,931 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:08:22,616 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:08:22,794 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:08:22,795 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:08:22,795 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:08:22,796 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:08:23,247 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:08:25,778 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:08:25,843 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:08:25,849 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:08:25,849 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:08:26,248 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:08:30,951 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:08:35,952 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:08:40,843 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:08:40,850 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:08:40,850 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:08:40,975 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:08:44,197 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:08:44,199 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:08:44,199 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:08:44,199 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:08:44,255 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:08:46,256 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:08:46,933 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:08:51,934 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:08:52,617 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:08:55,844 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:08:55,850 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:08:55,850 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:08:57,928 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:09:02,929 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:09:07,756 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:09:07,757 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:09:07,758 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:09:07,758 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:09:08,265 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:09:08,557 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:09:10,265 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:09:10,844 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:09:10,850 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:09:10,851 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:09:13,948 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:09:18,950 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:09:22,618 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:09:24,619 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:09:25,844 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:09:25,850 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:09:25,850 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:09:27,416 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:09:27,418 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:09:27,418 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:09:27,418 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:09:28,272 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:09:30,204 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:09:30,272 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:09:35,205 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:09:40,206 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:09:40,844 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:09:40,850 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:09:40,851 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:09:45,908 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:09:47,983 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:09:47,984 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:09:47,984 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:09:47,985 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:09:48,279 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:09:50,280 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:09:51,741 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:09:52,623 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:09:55,844 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:09:55,851 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:09:55,851 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:09:56,918 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:10:01,919 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:10:06,920 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:10:10,664 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:10:10,665 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:10:10,666 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:10:10,666 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:10:11,144 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:10:11,145 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:10:11,149 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:10:11,288 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:10:12,288 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:10:12,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:10:17,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:10:22,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:10:22,620 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:10:26,100 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:10:26,100 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:10:26,101 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:10:28,183 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:10:32,677 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:10:32,679 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:10:32,679 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:10:32,681 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:10:33,296 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:10:33,411 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:10:34,297 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:10:38,412 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:10:41,101 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:10:41,102 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:10:41,102 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:10:44,225 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:10:49,226 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:10:52,621 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:10:54,622 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:10:56,100 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:10:56,101 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:10:56,101 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:10:56,264 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:10:56,265 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:10:56,265 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:10:56,266 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:10:56,305 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:10:58,305 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:11:00,052 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:11:05,053 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:11:10,054 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:11:11,100 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:11:11,100 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:11:11,104 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:11:15,004 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:11:15,005 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:11:15,005 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:11:15,008 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:11:15,312 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:11:15,747 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:11:16,312 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:11:20,748 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:11:22,622 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:11:26,101 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:11:26,102 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:11:26,102 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:11:26,102 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:11:31,174 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:11:35,828 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:11:35,830 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:11:35,830 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:11:35,830 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:11:36,320 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:11:36,554 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:11:38,321 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:11:41,101 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:11:41,102 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:11:41,102 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:11:42,151 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:11:47,152 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:11:52,153 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:11:52,623 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:11:53,897 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:11:53,898 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:11:53,898 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:11:53,899 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:11:54,327 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:11:56,107 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:11:56,108 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:11:56,108 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:11:56,327 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:11:57,236 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:12:02,238 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:12:07,238 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:12:11,101 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:12:11,101 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:12:11,102 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:12:13,151 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:12:17,317 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:12:17,319 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:12:17,319 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:12:17,319 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:12:17,335 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:12:19,048 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:12:20,337 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:12:22,624 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:12:24,625 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:12:26,101 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:12:26,102 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:12:26,102 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:12:30,187 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:12:35,188 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:12:36,990 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:12:36,991 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:12:36,991 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:12:36,992 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:12:37,343 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:12:38,344 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:12:40,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:12:41,103 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:12:41,103 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:12:41,104 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:12:46,198 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:12:51,199 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:12:52,625 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:12:56,103 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:12:56,103 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:12:56,144 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:12:57,177 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:12:58,286 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:12:58,287 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:12:58,287 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:12:58,288 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:12:58,351 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:13:00,352 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:13:03,023 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:13:08,024 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:13:11,102 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:13:11,103 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:13:11,143 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:13:13,219 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:13:17,343 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:13:17,344 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:13:17,344 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:13:17,345 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:13:17,358 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:13:18,358 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:13:18,818 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:13:22,626 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:13:24,627 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:13:26,102 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:13:26,103 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:13:26,105 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:13:30,181 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:13:35,182 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:13:37,082 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:13:37,083 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:13:37,083 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:13:37,084 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:13:37,365 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:13:38,366 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:13:40,802 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:13:41,103 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:13:41,103 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:13:41,104 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:13:46,155 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:13:51,157 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:13:52,627 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:13:56,102 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:13:56,102 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:13:56,142 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:13:56,226 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:13:59,346 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:13:59,347 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:13:59,347 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:13:59,347 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:13:59,373 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:14:01,482 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:14:02,375 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:14:06,483 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:14:11,104 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:14:11,104 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:14:11,106 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:14:12,143 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:14:17,144 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:14:19,605 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:14:19,606 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:14:19,606 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:14:19,607 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:14:20,381 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:14:22,346 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:14:22,382 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:14:22,628 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:14:26,104 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:14:26,105 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:14:26,107 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:14:28,211 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:14:33,212 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:14:38,212 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:14:41,103 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:14:41,103 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:14:41,144 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:14:42,318 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:14:42,320 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:14:42,320 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:14:42,320 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:14:42,389 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:14:44,071 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:14:44,390 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:14:49,072 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:14:52,629 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:14:54,630 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:14:56,103 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:14:56,104 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:14:56,104 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:15:00,213 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:15:03,579 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:15:03,580 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:15:03,580 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:15:03,581 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:15:04,397 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:15:05,337 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:15:06,401 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:15:10,338 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:15:11,103 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:15:11,104 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:15:11,104 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:15:16,209 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:15:21,211 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:15:22,630 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:15:25,079 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:15:25,081 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:15:25,081 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:15:25,081 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:15:25,408 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:15:26,104 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:15:26,104 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:15:26,104 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:15:26,297 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:15:27,409 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:15:31,297 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:15:36,298 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:15:41,104 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:15:41,104 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:15:41,104 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:15:42,172 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:15:47,173 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:15:51,030 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:15:51,031 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:15:51,031 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:15:51,031 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:15:51,417 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:15:52,631 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:15:52,631 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:15:53,418 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:15:56,104 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:15:56,104 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:15:56,104 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:15:58,178 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:16:03,179 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:16:08,181 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:16:10,780 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:16:10,781 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:16:10,781 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:16:10,781 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:16:11,225 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:16:11,226 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:16:11,227 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:16:11,424 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:16:13,425 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:16:13,507 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:16:18,508 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:16:22,632 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:16:23,633 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:16:26,177 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:16:26,178 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:16:26,183 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:16:29,239 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:16:31,389 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:16:31,391 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:16:31,391 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:16:31,391 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:16:31,432 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:16:34,420 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:16:35,434 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:16:39,421 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:16:41,178 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:16:41,178 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:16:41,183 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:16:45,312 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:16:50,313 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:16:50,809 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:16:50,811 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:16:50,811 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:16:50,811 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:16:51,439 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:16:52,633 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:16:53,440 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:16:55,634 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:16:56,178 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:16:56,179 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:16:56,184 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:17:01,306 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:17:06,306 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:17:11,178 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:17:11,178 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:17:11,219 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:17:12,275 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:17:12,430 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:17:12,431 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:17:12,431 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:17:12,431 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:17:12,448 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:17:15,449 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:17:18,154 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:17:22,633 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:17:23,634 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:17:26,179 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:17:26,179 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:17:26,184 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:17:29,287 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:17:32,400 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:17:32,401 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:17:32,401 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:17:32,402 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:17:32,455 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:17:34,858 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:17:35,456 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:17:39,859 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:17:41,178 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:17:41,179 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:17:41,184 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:17:45,283 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:17:50,284 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:17:52,634 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:17:54,378 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:17:54,379 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:17:54,379 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:17:54,380 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:17:54,463 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:17:56,178 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:17:56,179 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:17:56,184 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:17:56,245 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:17:57,464 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:18:01,253 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:18:06,254 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:18:11,178 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:18:11,179 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:18:11,219 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:18:11,267 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:18:16,268 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:18:16,543 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:18:16,548 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:18:16,548 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:18:16,549 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:18:17,471 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:18:19,472 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:18:21,288 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:18:22,635 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:18:26,179 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:18:26,179 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:18:26,184 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:18:27,222 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:18:32,223 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:18:36,213 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:18:36,215 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:18:36,215 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:18:36,215 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:18:36,478 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:18:37,691 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:18:39,479 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:18:41,179 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:18:41,180 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:18:41,185 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:18:43,284 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:18:48,285 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:18:52,636 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:18:53,637 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:18:56,179 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:18:56,179 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:18:56,220 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:18:57,058 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:18:57,059 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:18:57,059 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:18:57,060 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:18:57,486 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:18:58,795 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:18:59,486 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:19:03,796 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:19:08,797 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:19:11,179 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:19:11,180 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:19:11,185 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:19:14,296 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:19:19,297 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:19:21,312 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:19:21,313 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:19:21,314 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:19:21,314 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:19:21,495 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:19:22,637 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:19:23,496 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:19:24,638 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:19:26,180 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:19:26,180 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:19:26,185 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:19:30,261 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:19:35,262 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:19:40,263 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:19:41,179 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:19:41,180 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:19:41,220 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:19:42,285 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:19:42,286 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:19:42,286 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:19:42,286 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:19:42,503 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:19:45,504 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:19:45,740 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:19:50,741 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:19:52,638 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:19:56,180 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:19:56,181 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:19:56,185 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:19:56,274 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:20:01,275 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:20:03,070 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:20:03,071 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:20:03,072 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:20:03,072 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:20:03,510 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:20:05,511 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:20:06,795 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:20:11,180 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:20:11,180 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:20:11,185 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:20:12,300 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:20:17,301 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:20:22,302 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:20:22,639 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:20:23,087 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:20:23,088 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:20:23,089 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:20:23,089 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:20:23,518 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:20:25,519 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:20:26,180 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:20:26,181 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:20:26,186 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:20:28,279 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:20:33,280 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:20:38,281 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:20:41,180 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:20:41,180 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:20:41,221 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:20:42,933 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:20:42,934 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:20:42,935 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:20:42,935 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:20:43,526 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:20:43,664 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:20:45,527 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:20:48,665 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:20:52,639 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:20:54,641 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:20:56,180 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:20:56,181 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:20:56,186 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:21:00,329 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:21:03,043 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:21:03,044 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:21:03,044 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:21:03,045 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:21:03,533 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:21:05,534 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:21:05,775 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:21:10,776 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:21:11,180 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:21:11,181 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:21:11,186 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:21:16,312 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:21:21,313 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:21:22,640 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:21:23,924 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:21:23,926 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:21:23,926 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:21:23,926 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:21:24,542 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:21:26,180 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:21:26,181 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:21:26,186 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:21:27,227 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:21:27,544 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:21:32,228 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:21:37,229 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:21:41,183 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:21:41,184 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:21:41,224 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:21:42,260 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:21:44,699 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:21:44,701 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:21:44,701 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:21:44,702 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:21:45,550 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:21:47,421 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:21:47,551 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:21:52,422 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:21:52,641 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:21:56,181 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:21:56,181 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:21:56,186 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:21:58,253 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:22:03,254 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:22:04,501 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:22:04,502 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:22:04,503 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:22:04,503 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:22:04,557 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:22:07,558 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:22:09,238 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:22:11,181 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:22:11,181 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:22:11,186 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:22:14,269 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:22:19,270 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:22:22,642 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:22:24,643 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:22:24,643 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:22:24,645 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:22:24,645 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:22:24,645 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:22:25,565 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:22:26,181 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:22:26,182 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:22:26,187 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:22:27,565 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:22:30,297 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:22:35,298 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:22:40,299 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:22:41,181 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:22:41,182 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:22:41,222 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:22:44,955 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:22:44,956 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:22:44,956 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:22:44,956 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:22:45,573 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:22:45,957 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:22:47,575 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:22:51,056 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:22:52,643 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:22:56,182 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:22:56,182 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:22:56,187 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:22:56,269 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:23:01,270 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:23:06,271 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:23:06,341 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:23:06,343 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:23:06,343 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:23:06,343 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:23:06,582 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:23:09,583 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:23:11,184 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:23:11,184 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:23:11,188 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:23:12,232 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:23:17,234 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:23:22,234 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:23:22,644 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:23:26,181 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:23:26,182 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:23:26,222 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:23:26,876 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:23:26,877 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:23:26,877 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:23:26,877 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:23:27,591 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:23:27,598 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:23:29,592 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:23:32,599 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:23:37,600 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:23:41,182 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:23:41,182 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:23:41,187 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:23:43,271 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:23:48,272 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:23:50,465 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:23:50,467 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:23:50,467 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:23:50,468 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:23:50,600 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:23:52,645 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:23:53,601 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:23:53,646 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:23:56,182 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:23:56,182 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:23:56,187 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:23:59,233 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:24:04,234 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:24:09,235 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:24:11,074 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:24:11,075 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:24:11,076 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:24:11,076 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:24:11,182 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:24:11,182 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:24:11,187 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:24:11,609 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:24:13,609 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:24:14,403 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:24:19,404 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:24:22,646 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:24:24,647 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:24:26,182 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:24:26,183 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:24:26,187 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:24:30,214 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:24:32,408 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:24:32,409 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:24:32,409 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:24:32,410 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:24:32,616 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:24:35,226 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:24:35,617 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:24:40,227 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:24:41,182 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:24:41,183 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:24:41,188 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:24:45,227 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:24:50,229 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:24:52,647 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:24:55,204 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:24:55,205 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:24:55,205 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:24:55,206 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:24:55,625 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:24:55,932 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:24:56,182 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:24:56,183 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:24:56,188 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:24:57,626 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:25:01,290 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:25:06,291 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:25:11,182 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:25:11,183 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:25:11,225 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:25:12,277 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:25:16,184 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:25:16,185 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:25:16,185 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:25:16,186 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:25:16,633 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:25:18,009 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:25:19,634 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:25:22,647 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:25:23,648 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:25:26,183 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:25:26,183 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:25:26,188 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:25:29,262 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:25:34,263 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:25:38,107 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:25:38,109 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:25:38,109 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:25:38,110 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:25:38,641 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:25:39,382 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:25:41,183 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:25:41,183 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:25:41,188 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:25:41,642 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:25:45,214 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:25:50,214 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:25:52,648 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:25:55,649 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:25:56,183 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:25:56,183 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:25:56,224 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:26:01,315 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:26:01,595 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:26:01,596 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:26:01,597 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:26:01,597 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:26:01,650 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:26:03,650 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:26:06,338 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:26:11,183 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:26:11,184 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:26:11,188 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:26:12,244 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:26:17,245 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:26:21,733 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:26:21,734 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:26:21,735 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:26:21,735 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:26:22,500 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:26:22,653 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:26:22,657 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:26:23,658 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:26:26,183 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:26:26,184 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:26:26,188 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:26:28,298 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:26:33,299 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:26:38,300 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:26:41,183 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:26:41,184 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:26:41,224 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:26:44,228 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:26:45,689 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:26:45,690 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:26:45,690 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:26:45,690 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:26:46,666 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:26:47,666 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:26:49,435 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:26:52,650 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:26:54,651 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:26:56,184 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:26:56,185 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:26:56,189 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:27:00,312 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:27:05,313 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:27:07,103 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:27:07,104 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:27:07,105 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:27:07,107 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:27:07,675 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:27:09,675 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:27:10,851 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:27:11,184 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:27:11,184 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:27:11,189 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:27:16,300 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:27:21,301 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:27:22,651 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:27:25,912 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:27:25,913 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:27:25,914 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:27:25,914 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:27:26,367 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:27:26,367 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:27:26,367 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:27:26,368 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:27:26,682 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:27:27,682 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:27:31,678 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:27:36,679 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:27:41,314 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:27:41,315 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:27:41,316 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:27:42,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:27:47,449 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:27:47,845 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:27:47,846 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:27:47,846 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:27:47,848 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:27:48,690 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:27:49,691 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:27:52,569 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:27:52,660 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:27:56,314 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:27:56,315 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:27:56,316 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:27:58,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:28:03,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:28:08,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:28:09,378 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:28:09,379 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:28:09,380 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:28:09,380 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:28:09,700 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:28:11,314 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:28:11,315 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:28:11,316 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:28:11,700 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:28:14,429 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:28:19,430 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:28:22,661 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:28:24,661 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:28:26,315 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:28:26,315 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:28:26,316 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:28:30,372 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:28:33,690 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:28:33,692 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:28:33,692 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:28:33,692 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:28:33,708 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:28:35,417 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:28:35,709 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:28:40,418 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:28:41,315 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:28:41,315 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:28:41,316 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:28:45,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:28:50,432 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:28:52,661 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:28:55,453 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:28:55,454 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:28:55,455 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:28:55,455 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:28:55,456 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:28:55,717 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:28:56,315 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:28:56,315 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:28:56,316 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:28:57,718 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:29:01,375 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:29:06,376 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:29:11,317 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:29:11,317 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:29:11,318 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:29:11,383 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:29:16,384 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:29:18,663 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:29:18,664 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:29:18,664 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:29:18,664 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:29:18,726 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:29:21,419 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:29:21,727 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:29:22,662 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:29:26,315 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:29:26,315 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:29:26,318 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:29:27,406 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:29:32,407 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:29:37,408 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:29:40,885 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:29:40,886 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:29:40,886 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:29:40,887 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:29:41,315 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:29:41,315 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:29:41,316 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:29:41,735 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:29:43,008 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:29:43,736 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:29:48,009 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:29:52,663 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:29:53,664 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:29:56,315 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:29:56,316 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:29:56,316 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:29:59,382 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:30:01,349 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:30:01,350 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:30:01,350 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:30:01,351 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:30:01,743 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:30:03,744 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:30:05,094 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:30:10,095 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:30:11,315 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:30:11,316 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:30:11,319 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:30:15,415 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:30:20,416 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:30:20,561 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:30:20,562 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:30:20,562 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:30:20,562 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:30:20,751 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:30:21,751 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:30:22,665 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:30:25,666 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:30:26,315 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:30:26,316 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:30:26,316 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:30:31,400 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:30:36,402 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:30:40,698 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:30:40,699 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:30:40,700 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:30:40,700 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:30:40,758 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:30:41,435 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:30:41,435 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:30:41,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:30:41,543 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:30:43,759 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:30:46,544 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:30:51,545 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:30:52,665 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:30:56,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:30:56,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:30:56,443 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:30:57,504 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:31:02,505 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:31:02,944 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:31:02,946 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:31:02,946 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:31:02,946 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:31:03,767 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:31:05,768 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:31:07,673 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:31:11,437 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:31:11,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:31:11,438 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:31:13,474 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:31:18,475 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:31:22,667 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:31:23,667 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:31:23,921 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:31:23,923 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:31:23,923 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:31:23,923 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:31:24,776 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:31:25,776 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:31:26,437 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:31:26,437 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:31:26,438 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:31:29,522 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:31:34,523 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:31:39,524 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:31:41,437 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:31:41,437 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:31:41,477 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:31:45,487 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:31:45,648 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:31:45,649 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:31:45,650 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:31:45,650 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:31:45,784 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:31:47,785 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:31:51,403 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:31:52,668 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:31:56,437 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:31:56,437 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:31:56,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:31:56,438 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:32:01,569 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:32:06,571 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:32:07,847 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:32:07,848 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:32:07,848 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:32:07,849 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:32:08,792 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:32:09,793 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:32:11,437 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:32:11,437 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:32:11,438 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:32:12,546 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:32:17,546 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:32:22,547 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:32:22,668 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:32:26,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:32:26,438 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:32:26,478 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:32:27,560 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:32:28,966 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:32:28,968 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:32:28,968 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:32:28,968 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:32:29,800 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:32:31,801 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:32:32,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:32:37,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:32:41,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:32:41,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:32:41,440 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:32:43,473 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:32:48,474 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:32:50,002 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:32:50,003 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:32:50,003 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:32:50,004 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:32:50,808 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:32:51,809 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:32:52,669 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:32:53,670 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:32:56,437 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:32:56,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:32:56,438 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:32:59,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:33:04,574 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:33:09,575 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:33:10,452 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:33:10,454 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:33:10,454 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:33:10,454 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:33:10,816 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:33:11,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:33:11,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:33:11,440 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:33:11,816 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:33:15,546 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:33:20,547 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:33:22,670 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:33:25,671 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:33:26,437 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:33:26,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:33:26,438 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:33:31,541 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:33:34,824 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:33:34,825 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:33:34,826 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:33:34,826 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:33:35,827 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:33:36,714 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:33:37,828 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:33:41,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:33:41,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:33:41,439 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:33:42,562 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:33:47,563 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:33:52,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:33:52,671 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:33:53,767 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:33:53,768 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:33:53,768 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:33:53,768 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:33:53,835 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:33:55,836 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:33:56,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:33:56,439 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:33:56,479 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:33:58,483 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:34:03,484 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:34:08,485 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:34:11,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:34:11,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:34:11,440 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:34:14,485 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:34:15,241 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:34:15,242 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:34:15,242 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:34:15,242 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:34:15,844 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:34:17,844 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:34:19,998 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:34:22,672 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:34:25,673 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:34:26,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:34:26,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:34:26,439 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:34:31,486 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:34:36,487 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:34:37,510 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:34:37,512 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:34:37,512 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:34:37,512 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:34:37,852 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:34:39,853 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:34:41,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:34:41,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:34:41,440 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:34:42,464 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:34:47,465 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:34:52,466 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:34:52,673 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:34:56,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:34:56,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:34:56,440 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:34:57,531 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:34:59,502 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:34:59,504 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:34:59,504 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:34:59,504 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:34:59,861 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:35:01,862 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:35:03,230 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:35:08,231 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:35:11,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:35:11,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:35:11,440 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:35:13,548 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:35:18,549 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:35:21,349 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:35:21,350 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:35:21,350 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:35:21,351 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:35:21,869 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:35:22,673 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:35:23,674 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:35:23,870 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:35:26,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:35:26,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:35:26,440 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:35:29,563 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:35:34,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:35:39,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:35:41,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:35:41,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:35:41,441 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:35:42,025 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:35:42,026 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:35:42,026 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:35:42,027 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:35:42,877 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:35:43,877 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:35:44,752 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:35:49,753 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:35:52,674 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:35:55,675 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:35:56,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:35:56,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:35:56,440 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:36:01,473 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:36:03,964 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:36:03,965 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:36:03,965 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:36:03,966 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:36:04,885 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:36:05,885 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:36:06,932 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:36:11,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:36:11,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:36:11,441 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:36:12,475 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:36:17,476 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:36:22,476 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:36:22,675 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:36:24,239 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:36:24,240 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:36:24,240 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:36:24,241 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:36:24,893 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:36:25,893 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:36:26,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:36:26,442 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:36:26,482 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:36:27,562 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:36:32,563 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:36:37,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:36:41,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:36:41,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:36:41,441 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:36:43,559 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:36:47,698 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:36:47,699 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:36:47,699 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:36:47,699 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:36:47,902 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:36:48,700 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:36:49,903 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:36:52,676 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:36:54,677 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:36:56,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:36:56,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:36:56,445 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:37:00,525 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:37:05,526 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:37:07,870 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:37:07,872 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:37:07,872 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:37:07,872 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:37:07,910 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:37:09,911 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:37:10,612 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:37:11,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:37:11,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:37:11,441 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:37:16,514 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:37:21,515 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:37:22,677 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:37:26,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:37:26,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:37:26,441 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:37:26,536 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:37:27,626 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:37:27,628 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:37:27,628 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:37:27,628 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:37:27,917 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:37:29,918 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:37:32,371 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:37:37,372 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:37:41,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:37:41,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:37:41,442 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:37:42,484 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:37:47,486 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:37:47,544 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:37:47,545 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:37:47,545 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:37:47,546 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:37:47,924 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:37:50,926 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:37:52,678 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:37:52,679 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:37:56,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:37:56,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:37:56,442 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:37:58,543 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:38:03,544 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:38:08,545 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:38:10,326 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:38:10,327 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:38:10,327 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:38:10,328 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:38:10,934 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:38:11,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:38:11,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:38:11,443 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:38:12,935 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:38:13,557 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:38:18,558 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:38:22,679 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:38:23,680 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:38:26,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:38:26,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:38:26,442 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:38:29,562 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:38:32,180 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:38:32,182 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:38:32,182 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:38:32,182 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:38:32,942 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:38:34,941 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:38:34,942 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:38:39,942 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:38:41,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:38:41,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:38:41,442 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:38:45,546 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:38:50,546 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:38:52,233 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:38:52,234 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:38:52,234 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:38:52,235 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:38:52,680 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:38:52,949 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:38:54,950 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:38:56,010 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:38:56,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:38:56,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:38:56,442 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:39:01,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:39:06,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:39:11,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:39:11,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:39:11,442 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:39:12,471 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:39:14,342 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:39:14,343 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:39:14,343 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:39:14,344 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:39:14,958 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:39:16,959 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:39:18,394 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:39:22,681 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:39:23,682 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:39:26,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:39:26,445 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:39:26,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:39:29,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:39:34,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:39:35,894 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:39:35,895 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:39:35,895 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:39:35,896 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:39:35,966 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:39:38,967 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:39:40,406 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:39:41,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:39:41,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:39:41,446 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:39:45,562 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:39:50,563 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:39:52,682 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:39:55,683 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:39:56,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:39:56,444 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:39:56,485 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:39:57,906 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:39:57,908 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:39:57,908 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:39:57,908 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:39:57,974 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:40:00,975 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:40:01,141 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:40:06,142 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:40:11,143 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:40:11,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:40:11,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:40:11,445 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:40:16,577 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:40:18,796 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:40:18,797 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:40:18,797 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:40:18,798 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:40:18,982 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:40:20,983 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:40:22,530 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:40:22,683 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:40:26,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:40:26,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:40:26,446 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:40:28,515 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:40:33,516 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:40:38,489 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:40:38,491 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:40:38,491 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:40:38,491 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:40:38,991 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:40:39,205 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:40:40,992 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:40:41,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:40:41,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:40:41,445 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:40:44,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:40:49,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:40:52,684 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:40:54,685 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:40:56,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:40:56,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:40:56,446 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:40:58,236 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:40:58,237 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:40:58,238 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:40:58,238 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:40:58,999 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:40:59,991 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:41:00,999 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:41:04,992 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:41:09,993 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:41:11,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:41:11,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:41:11,445 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:41:15,557 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:41:18,093 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:41:18,094 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:41:18,094 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:41:18,095 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:41:19,006 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:41:20,831 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:41:21,007 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:41:22,685 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:41:26,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:41:26,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:41:26,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:41:26,446 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:41:31,555 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:41:36,556 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:41:37,940 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:41:37,941 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:41:37,941 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:41:37,942 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:41:38,014 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:41:41,015 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:41:41,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:41:41,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:41:41,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:41:41,574 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:41:46,575 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:41:51,576 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:41:52,686 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:41:56,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:41:56,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:41:56,445 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:41:57,484 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:42:00,252 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:42:00,253 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:42:00,253 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:42:00,254 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:42:01,023 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:42:03,025 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:42:03,035 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:42:08,036 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:42:11,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:42:11,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:42:11,446 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:42:13,578 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:42:18,579 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:42:19,775 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:42:19,776 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:42:19,776 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:42:19,777 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:42:20,032 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:42:22,687 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:42:23,033 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:42:23,687 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:42:26,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:42:26,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:42:26,446 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:42:29,541 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:42:34,542 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:42:39,543 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:42:41,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:42:41,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:42:41,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:42:41,721 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:42:41,722 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:42:41,722 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:42:41,723 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:42:42,040 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:42:44,902 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:42:45,042 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:42:49,903 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:42:52,687 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:42:55,689 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:42:56,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:42:56,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:42:56,446 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:43:01,560 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:43:03,782 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:43:03,783 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:43:03,784 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:43:03,784 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:43:04,048 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:43:06,664 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:43:07,050 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:43:11,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:43:11,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:43:11,446 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:43:12,511 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:43:17,512 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:43:22,513 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:43:22,688 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:43:23,955 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:43:23,956 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:43:23,956 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:43:23,957 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:43:24,056 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:43:26,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:43:26,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:43:26,446 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:43:27,057 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:43:27,555 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:43:32,557 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:43:37,557 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:43:41,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:43:41,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:43:41,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:43:42,558 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:43:43,288 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:43:43,290 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:43:43,290 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:43:43,290 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:43:44,064 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:43:45,064 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:43:48,030 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:43:52,689 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:43:53,690 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:43:56,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:43:56,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:43:56,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:43:59,577 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:44:02,254 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:44:02,256 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:44:02,256 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:44:02,256 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:44:03,071 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:44:04,988 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:44:05,072 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:44:09,990 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:44:11,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:44:11,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:44:11,448 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:44:15,521 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:44:20,522 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:44:20,927 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:44:20,928 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:44:20,928 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:44:20,929 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:44:21,077 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:44:22,690 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:44:23,078 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:44:25,691 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:44:26,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:44:26,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:44:26,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:44:31,488 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:44:36,489 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:44:41,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:44:41,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:44:41,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:44:41,581 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:44:43,127 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:44:43,129 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:44:43,129 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:44:43,129 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:44:44,085 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:44:45,086 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:44:46,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:44:51,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:44:52,691 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:44:56,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:44:56,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:44:56,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:44:57,534 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:45:02,535 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:45:04,097 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:45:04,098 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:45:04,098 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:45:04,099 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:45:05,094 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:45:07,095 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:45:07,886 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:45:11,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:45:11,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:45:11,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:45:13,571 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:45:18,572 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:45:22,692 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:45:23,693 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:45:26,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:45:26,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:45:26,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:45:26,907 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:45:26,908 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:45:26,908 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:45:26,909 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:45:27,102 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:45:28,847 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:45:29,103 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:45:33,848 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:45:38,848 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:45:41,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:45:41,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:45:41,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:45:44,528 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:45:47,913 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:45:47,914 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:45:47,915 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:45:47,915 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:45:48,110 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:45:49,677 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:45:51,111 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:45:52,693 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:45:54,694 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:45:56,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:45:56,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:45:56,448 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:46:00,502 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:46:05,503 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:46:08,357 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:46:08,358 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:46:08,358 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:46:08,359 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:46:09,119 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:46:11,120 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:46:11,122 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:46:11,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:46:11,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:46:11,448 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:46:16,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:46:21,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:46:22,694 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:46:26,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:46:26,449 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:46:26,450 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:46:26,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:46:30,508 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:46:30,509 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:46:30,510 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:46:30,510 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:46:31,128 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:46:32,239 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:46:33,129 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:46:37,240 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:46:41,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:46:41,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:46:41,448 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:46:42,527 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:46:47,528 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:46:51,603 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:46:51,604 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:46:51,604 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:46:51,605 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:46:52,137 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:46:52,606 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:46:52,695 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:46:55,139 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:46:56,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:46:56,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:46:56,448 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:46:58,571 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:47:03,572 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:47:08,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:47:11,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:47:11,450 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:47:11,450 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:47:14,489 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:47:15,002 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:47:15,003 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:47:15,004 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:47:15,004 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:47:15,147 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:47:17,148 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:47:19,841 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:47:22,696 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:47:25,697 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:47:26,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:47:26,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:47:26,448 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:47:31,555 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:47:36,556 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:47:37,070 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:47:37,071 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:47:37,072 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:47:37,072 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:47:37,156 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:47:39,157 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:47:41,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:47:41,450 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:47:41,450 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:47:42,512 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:47:47,513 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:47:52,514 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:47:52,697 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:47:56,449 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:47:56,450 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:47:56,450 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:47:57,583 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:47:59,127 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:47:59,128 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:47:59,129 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:47:59,129 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:47:59,164 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:48:01,165 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:48:02,889 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:48:07,890 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:48:11,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:48:11,450 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:48:11,450 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:48:13,485 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:48:18,486 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:48:19,870 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:48:19,871 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:48:19,872 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:48:19,872 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:48:20,172 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:48:22,697 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:48:23,174 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:48:23,698 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:48:26,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:48:26,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:48:26,449 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:48:29,528 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:48:34,529 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:48:39,530 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:48:39,681 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:48:39,682 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:48:39,682 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:48:39,683 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:48:40,180 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:48:41,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:48:41,449 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:48:41,449 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:48:43,181 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:48:44,539 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:48:49,541 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:48:52,698 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:48:54,699 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:48:56,455 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:48:56,456 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:48:56,456 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:49:00,584 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:49:00,629 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:49:00,630 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:49:00,631 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:49:00,631 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:49:01,188 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:49:03,189 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:49:06,362 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:49:11,363 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:49:11,453 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:49:11,454 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:49:11,454 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:49:16,504 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:49:21,505 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:49:21,838 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:49:21,839 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:49:21,840 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:49:21,840 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:49:22,196 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:49:22,699 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:49:25,198 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:49:26,454 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:49:26,455 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:49:26,455 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:49:26,517 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:49:31,518 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:49:36,519 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:49:41,019 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:49:41,020 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:49:41,020 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:49:41,021 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:49:41,204 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:49:41,478 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:49:41,478 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:49:41,479 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:49:41,534 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:49:43,205 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:49:46,771 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:49:51,772 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:49:52,700 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:49:56,459 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:49:56,460 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:49:56,460 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:49:57,537 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:50:02,538 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:50:02,924 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:50:02,925 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:50:02,925 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:50:02,925 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:50:03,213 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:50:05,214 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:50:07,677 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:50:11,459 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:50:11,459 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:50:11,460 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:50:13,594 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:50:18,595 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:50:22,701 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:50:23,702 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:50:24,401 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:50:24,402 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:50:24,403 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:50:24,403 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:50:25,221 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:50:26,460 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:50:26,460 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:50:26,461 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:50:27,222 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:50:29,544 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:50:34,545 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:50:39,546 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:50:41,459 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:50:41,461 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:50:41,462 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:50:44,547 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:50:45,948 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:50:45,949 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:50:45,950 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:50:45,950 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:50:46,230 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:50:49,231 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:50:49,812 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:50:52,706 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:50:55,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:50:56,458 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:50:56,459 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:50:56,459 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:51:01,586 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:51:06,587 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:51:07,161 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:51:07,163 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:51:07,163 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:51:07,163 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:51:07,237 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:51:09,238 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:51:11,459 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:51:11,459 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:51:11,463 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:51:12,545 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:51:17,546 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:51:22,547 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:51:22,702 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:51:26,459 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:51:26,459 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:51:26,500 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:51:28,529 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:51:33,530 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:51:34,313 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:51:34,314 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:51:34,314 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:51:34,316 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:51:35,247 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:51:37,248 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:51:39,044 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:51:41,459 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:51:41,459 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:51:41,460 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:51:44,560 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:51:49,561 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:51:52,703 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:51:54,704 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:51:55,067 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:51:55,068 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:51:55,069 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:51:55,069 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:51:55,255 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:51:56,459 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:51:56,459 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:51:56,460 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:51:57,256 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:52:00,585 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:52:05,586 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:52:10,587 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:52:11,459 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:52:11,460 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:52:11,460 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:52:16,106 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:52:16,107 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:52:16,107 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:52:16,109 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:52:16,110 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:52:16,262 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:52:19,263 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:52:21,869 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:52:22,704 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:52:26,459 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:52:26,460 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:52:26,461 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:52:27,493 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:52:32,494 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:52:35,358 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:52:35,360 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:52:35,360 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:52:35,360 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:52:36,269 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:52:37,270 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:52:38,111 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:52:41,459 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:52:41,460 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:52:41,461 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:52:43,599 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:52:48,600 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:52:52,705 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:52:53,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:52:56,320 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:52:56,322 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:52:56,322 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:52:56,323 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:52:56,459 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:52:56,460 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:52:56,461 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:52:57,278 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:52:59,278 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:52:59,541 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:53:04,542 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:53:09,543 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:53:11,460 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:53:11,461 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:53:11,461 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:53:14,554 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:53:19,233 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:53:19,235 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:53:19,235 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:53:19,235 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:53:19,286 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:53:19,991 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:53:21,286 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:53:22,707 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:53:25,708 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:53:26,460 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:53:26,461 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:53:26,461 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:53:31,501 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:53:36,503 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:53:37,123 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:53:37,124 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:53:37,125 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:53:37,125 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:53:37,292 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:53:39,293 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:53:41,460 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:53:41,460 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:53:41,461 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:53:41,513 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:53:46,513 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:53:51,514 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:53:52,707 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:53:54,465 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:53:54,467 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:53:54,467 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:53:54,467 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:53:55,299 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:53:56,462 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:53:56,463 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:53:56,463 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:53:56,533 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:53:57,300 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:54:01,534 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:54:06,534 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:54:11,461 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:54:11,462 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:54:11,462 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:54:12,533 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:54:15,003 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:54:15,004 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:54:15,004 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:54:15,005 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:54:15,307 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:54:17,308 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:54:17,737 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:54:22,708 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:54:23,709 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:54:26,461 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:54:26,461 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:54:26,462 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:54:29,551 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:54:34,020 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:54:34,021 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:54:34,021 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:54:34,022 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:54:34,314 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:54:34,758 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:54:37,315 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:54:39,758 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:54:41,462 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:54:41,463 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:54:41,466 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:54:45,514 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:54:50,515 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:54:52,710 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:54:53,486 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:54:53,487 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:54:53,487 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:54:53,487 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:54:54,320 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:54:55,321 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:54:56,247 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:54:56,461 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:54:56,462 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:54:56,462 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:55:01,587 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:55:06,587 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:55:11,461 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:55:11,462 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:55:11,463 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:55:11,592 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:55:13,822 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:55:13,823 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:55:13,823 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:55:13,824 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:55:14,329 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:55:15,329 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:55:17,545 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:55:22,546 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:55:22,710 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:55:26,466 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:55:26,466 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:55:26,469 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:55:27,591 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:55:32,592 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:55:35,949 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:55:35,950 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:55:35,951 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:55:35,951 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:55:36,338 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:55:37,338 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:55:37,700 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:55:41,466 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:55:41,467 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:55:41,468 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:55:43,513 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:55:48,514 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:55:52,711 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:55:53,712 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:55:54,758 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:55:54,760 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:55:54,760 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:55:54,760 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:55:55,345 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:55:56,562 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:55:56,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:55:56,564 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:55:57,346 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:55:59,655 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:56:04,656 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:56:09,657 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:56:11,560 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:56:11,561 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:56:11,562 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:56:14,359 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:56:14,360 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:56:14,361 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:56:14,361 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:56:15,118 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:56:15,352 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:56:17,353 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:56:20,118 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:56:22,713 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:56:25,714 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:56:26,559 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:56:26,560 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:56:26,560 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:56:31,597 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:56:35,956 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:56:35,958 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:56:35,958 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:56:35,958 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:56:36,360 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:56:36,742 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:56:37,361 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:56:41,559 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:56:41,559 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:56:41,562 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:56:42,627 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:56:47,628 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:56:52,629 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:56:52,714 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:56:54,748 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:56:54,749 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:56:54,749 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:56:54,750 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:56:55,367 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:56:56,558 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:56:56,559 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:56:56,563 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:56:57,368 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:56:57,669 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:57:02,670 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:57:07,671 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:57:11,561 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:57:11,561 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:57:11,562 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:57:13,630 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:57:15,017 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:57:15,019 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:57:15,019 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:57:15,019 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:57:15,375 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:57:17,375 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:57:18,781 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:57:22,715 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:57:24,716 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:57:26,559 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:57:26,559 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:57:26,560 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:57:30,598 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:57:35,599 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:57:37,465 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:57:37,466 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:57:37,466 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:57:37,467 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:57:38,383 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:57:39,384 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:57:41,421 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:57:41,561 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:57:41,562 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:57:41,562 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:57:46,672 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:57:51,673 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:57:52,716 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:57:56,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:57:56,565 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:57:56,605 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:57:57,612 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:57:58,409 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:57:58,410 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:57:58,411 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:57:58,411 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:57:59,391 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:58:01,392 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:58:03,150 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:58:08,151 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:58:11,563 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:58:11,563 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:58:11,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:58:13,693 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:58:17,996 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:58:17,997 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:58:17,998 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:58:17,998 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:58:18,399 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:58:18,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:58:19,400 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:58:22,717 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:58:24,718 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:58:26,562 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:58:26,562 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:58:26,563 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:58:30,641 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:58:35,642 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:58:40,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:58:40,574 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:58:40,574 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:58:40,574 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:58:41,303 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:58:41,407 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:58:41,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:58:41,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:58:41,569 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:58:43,408 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:58:46,653 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:58:51,654 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:58:52,718 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:58:56,563 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:58:56,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:58:56,568 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:58:57,650 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:58:59,546 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:58:59,552 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:58:59,552 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:58:59,552 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:59:00,414 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:59:01,414 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:59:03,287 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:59:08,288 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:59:11,563 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:59:11,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:59:11,566 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:59:13,600 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:59:18,601 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:59:19,088 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:59:19,089 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:59:19,090 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:59:19,090 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:59:19,421 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:59:21,422 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:59:22,719 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:59:23,720 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:59:26,563 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:59:26,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:59:26,567 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:59:29,646 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:59:34,647 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:59:37,867 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:59:37,868 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:59:37,868 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:59:37,869 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:59:38,428 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:59:39,428 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 14:59:40,639 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:59:41,569 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:59:41,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:59:41,570 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:59:45,670 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:59:50,671 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:59:52,720 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 14:59:55,721 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 14:59:56,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 14:59:56,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 14:59:56,567 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 14:59:57,928 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 14:59:57,929 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 14:59:57,929 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 14:59:57,930 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 14:59:58,435 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 14:59:59,438 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:00:01,666 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:00:06,667 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:00:11,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:00:11,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:00:11,567 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:00:11,674 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:00:16,675 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:00:19,641 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:00:19,642 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:00:19,643 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:00:19,643 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:00:20,446 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:00:22,389 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:00:22,447 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:00:22,721 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:00:26,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:00:26,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:00:26,567 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:00:27,654 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:00:32,655 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:00:37,656 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:00:41,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:00:41,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:00:41,567 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:00:43,625 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:00:44,670 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:00:44,671 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:00:44,672 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:00:44,672 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:00:45,457 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:00:48,458 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:00:49,405 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:00:52,722 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:00:54,723 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:00:56,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:00:56,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:00:56,567 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:01:00,639 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:01:05,640 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:01:06,178 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:01:06,179 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:01:06,179 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:01:06,180 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:01:06,464 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:01:08,465 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:01:10,974 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:01:11,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:01:11,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:01:11,567 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:01:16,685 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:01:21,686 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:01:22,723 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:01:24,334 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:01:24,335 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:01:24,335 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:01:24,336 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:01:24,470 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:01:26,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:01:26,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:01:26,568 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:01:27,641 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:01:28,472 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:01:32,642 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:01:37,643 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:01:41,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:01:41,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:01:41,567 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:01:42,666 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:01:47,667 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:01:49,993 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:01:49,994 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:01:49,995 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:01:49,995 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:01:50,479 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:01:52,480 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:01:52,721 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:01:52,724 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:01:56,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:01:56,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:01:56,568 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:01:58,659 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:02:03,659 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:02:08,661 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:02:10,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:02:10,736 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:02:10,736 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:02:10,737 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:02:11,487 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:02:11,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:02:11,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:02:11,568 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:02:13,719 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:02:14,488 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:02:18,721 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:02:22,725 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:02:23,726 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:02:26,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:02:26,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:02:26,568 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:02:29,695 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:02:30,586 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:02:30,588 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:02:30,588 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:02:30,589 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:02:31,495 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:02:34,496 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:02:35,328 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:02:40,329 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:02:41,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:02:41,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:02:41,568 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:02:45,639 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:02:50,640 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:02:51,763 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:02:51,764 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:02:51,764 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:02:51,765 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:02:52,502 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:02:52,727 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:02:54,503 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:02:55,728 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:02:56,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:02:56,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:02:56,569 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:03:01,679 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:03:06,680 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:03:11,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:03:11,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:03:11,569 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:03:12,637 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:03:13,174 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:03:13,176 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:03:13,176 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:03:13,176 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:03:13,512 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:03:16,513 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:03:17,914 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:03:22,728 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:03:23,728 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:03:26,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:03:26,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:03:26,573 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:03:29,714 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:03:34,716 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:03:35,371 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:03:35,373 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:03:35,373 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:03:35,373 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:03:35,519 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:03:38,520 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:03:40,117 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:03:41,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:03:41,569 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:03:41,569 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:03:45,634 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:03:50,635 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:03:52,729 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:03:54,755 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:03:54,756 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:03:54,756 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:03:54,757 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:03:55,527 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:03:56,497 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:03:56,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:03:56,569 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:03:56,569 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:03:58,528 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:04:01,637 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:04:06,638 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:04:11,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:04:11,569 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:04:11,569 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:04:11,691 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:04:16,691 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:04:18,357 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:04:18,359 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:04:18,359 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:04:18,359 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:04:18,535 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:04:20,536 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:04:22,125 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:04:22,730 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:04:26,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:04:26,569 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:04:26,569 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:04:27,678 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:04:32,679 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:04:37,679 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:04:40,272 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:04:40,273 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:04:40,273 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:04:40,274 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:04:40,544 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:04:41,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:04:41,569 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:04:41,569 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:04:42,544 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:04:42,690 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:04:47,691 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:04:52,692 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:04:52,730 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:04:56,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:04:56,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:04:56,570 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:04:57,701 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:05:02,702 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:05:02,743 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:05:02,744 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:05:02,744 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:05:02,745 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:05:03,553 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:05:06,554 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:05:08,523 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:05:11,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:05:11,569 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:05:11,570 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:05:13,612 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:05:18,613 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:05:22,732 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:05:23,732 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:05:24,655 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:05:24,656 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:05:24,656 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:05:24,657 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:05:25,563 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:05:26,563 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:05:26,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:05:26,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:05:26,570 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:05:29,701 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:05:34,702 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:05:39,703 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:05:41,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:05:41,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:05:41,570 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:05:45,324 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:05:45,325 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:05:45,325 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:05:45,326 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:05:45,326 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:05:45,571 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:05:48,572 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:05:51,072 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:05:52,733 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:05:56,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:05:56,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:05:56,571 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:05:56,678 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:06:01,679 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:06:06,591 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:06:06,593 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:06:06,593 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:06:06,594 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:06:07,517 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:06:07,580 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:06:10,581 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:06:11,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:06:11,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:06:11,570 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:06:12,610 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:06:17,611 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:06:22,612 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:06:22,734 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:06:26,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:06:26,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:06:26,571 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:06:27,671 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:06:28,738 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:06:28,739 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:06:28,739 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:06:28,740 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:06:29,589 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:06:30,589 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:06:33,458 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:06:38,459 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:06:41,569 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:06:41,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:06:41,570 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:06:43,679 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:06:48,680 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:06:49,616 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:06:49,617 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:06:49,617 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:06:49,618 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:06:50,597 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:06:52,597 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:06:52,735 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:06:53,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:06:56,569 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:06:56,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:06:56,571 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:06:59,658 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:07:04,659 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:07:09,660 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:07:10,123 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:07:10,124 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:07:10,125 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:07:10,125 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:07:10,604 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:07:11,569 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:07:11,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:07:11,571 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:07:12,604 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:07:14,699 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:07:19,700 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:07:22,741 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:07:24,741 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:07:26,572 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:07:26,572 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:07:26,573 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:07:30,624 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:07:32,814 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:07:32,816 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:07:32,816 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:07:32,816 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:07:33,613 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:07:34,613 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:07:36,603 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:07:41,569 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:07:41,571 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:07:41,571 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:07:41,605 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:07:46,606 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:07:51,607 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:07:52,737 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:07:54,531 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:07:54,532 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:07:54,532 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:07:54,534 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:07:54,621 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:07:56,569 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:07:56,571 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:07:56,572 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:07:56,622 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:07:56,701 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:08:01,701 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:08:06,702 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:08:11,569 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:08:11,571 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:08:11,571 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:08:12,685 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:08:16,638 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:08:16,639 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:08:16,639 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:08:16,639 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:08:17,629 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:08:18,401 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:08:18,629 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:08:22,737 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:08:23,738 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:08:26,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:08:26,572 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:08:26,572 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:08:29,644 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:08:34,645 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:08:35,656 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:08:35,657 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:08:35,657 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:08:35,659 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:08:36,637 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:08:38,637 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:08:40,384 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:08:41,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:08:41,571 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:08:41,572 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:08:45,759 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:08:50,760 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:08:52,738 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:08:55,678 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:08:55,681 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:08:55,682 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:08:55,682 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:08:56,398 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:08:56,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:08:56,572 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:08:56,572 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:08:56,643 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:08:58,644 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:09:01,609 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:09:06,609 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:09:11,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:09:11,571 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:09:11,572 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:09:11,646 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:09:16,647 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:09:18,334 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:09:18,335 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:09:18,335 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:09:18,337 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:09:18,652 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:09:20,653 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:09:22,077 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:09:22,739 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:09:26,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:09:26,572 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:09:26,573 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:09:27,696 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:09:32,697 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:09:36,958 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:09:36,959 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:09:36,959 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:09:36,959 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:09:37,659 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:09:37,718 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:09:38,660 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:09:41,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:09:41,572 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:09:41,572 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:09:43,648 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:09:48,649 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:09:52,740 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:09:53,741 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:09:56,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:09:56,572 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:09:56,573 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:09:58,185 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:09:58,187 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:09:58,187 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:09:58,187 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:09:58,667 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:09:58,923 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:10:00,668 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:10:03,924 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:10:08,925 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:10:11,571 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:10:11,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:10:11,573 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:10:14,606 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:10:17,602 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:10:17,604 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:10:17,604 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:10:17,604 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:10:17,675 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:10:20,339 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:10:20,676 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:10:22,741 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:10:25,742 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:10:26,571 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:10:26,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:10:26,573 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:10:31,678 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:10:35,357 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:10:35,359 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:10:35,359 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:10:35,360 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:10:35,682 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:10:37,091 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:10:38,683 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:10:41,571 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:10:41,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:10:41,573 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:10:42,614 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:10:47,615 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:10:52,616 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:10:52,741 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:10:56,571 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:10:56,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:10:56,574 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:10:57,704 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:11:01,983 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:11:01,984 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:11:01,984 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:11:01,985 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:11:02,694 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:11:02,716 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:11:04,695 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:11:07,716 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:11:11,571 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:11:11,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:11:11,573 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:11:13,643 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:11:18,644 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:11:22,742 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:11:22,807 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:11:22,808 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:11:22,808 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:11:22,808 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:11:23,702 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:11:24,549 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:11:24,702 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:11:26,572 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:11:26,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:11:26,574 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:11:29,596 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:11:34,597 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:11:39,598 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:11:41,571 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:11:41,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:11:41,574 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:11:42,588 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:11:42,590 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:11:42,590 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:11:42,591 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:11:42,709 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:11:44,710 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:11:45,369 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:11:50,370 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:11:52,743 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:11:55,744 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:11:56,572 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:11:56,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:11:56,574 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:12:01,708 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:12:04,071 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:12:04,072 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:12:04,072 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:12:04,073 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:12:04,718 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:12:06,719 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:12:06,828 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:12:11,572 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:12:11,574 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:12:11,574 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:12:12,681 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:12:17,682 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:12:22,683 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:12:22,744 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:12:26,572 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:12:26,574 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:12:26,575 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:12:27,265 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:12:27,266 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:12:27,267 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:12:27,267 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:12:27,728 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:12:28,013 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:12:30,729 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:12:33,014 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:12:38,015 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:12:41,572 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:12:41,574 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:12:41,574 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:12:43,701 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:12:48,702 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:12:51,906 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:12:51,907 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:12:51,908 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:12:51,908 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:12:52,738 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:12:52,745 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:12:53,745 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:12:54,739 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:12:56,572 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:12:56,574 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:12:56,575 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:12:59,636 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:13:04,637 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:13:09,638 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:13:11,572 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:13:11,574 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:13:11,574 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:13:14,641 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:13:14,895 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:13:14,896 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:13:14,897 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:13:14,897 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:13:15,747 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:13:16,748 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:13:19,648 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:13:22,745 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:13:24,746 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:13:26,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:13:26,575 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:13:26,575 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:13:30,695 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:13:35,235 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:13:35,236 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:13:35,236 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:13:35,237 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:13:35,755 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:13:35,976 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:13:36,755 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:13:40,977 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:13:41,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:13:41,574 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:13:41,575 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:13:46,600 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:13:51,600 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:13:52,746 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:13:54,066 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:13:54,067 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:13:54,068 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:13:54,068 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:13:54,762 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:13:56,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:13:56,575 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:13:56,575 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:13:56,708 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:13:56,763 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:14:01,709 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:14:06,710 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:14:11,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:14:11,575 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:14:11,575 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:14:12,694 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:14:13,870 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:14:13,872 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:14:13,872 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:14:13,872 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:14:14,770 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:14:16,771 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:14:18,610 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:14:22,747 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:14:23,748 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:14:26,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:14:26,575 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:14:26,575 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:14:29,621 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:14:33,959 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:14:33,960 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:14:33,961 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:14:33,961 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:14:34,704 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:14:34,778 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:14:36,778 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:14:39,705 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:14:41,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:14:41,575 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:14:41,575 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:14:45,681 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:14:50,682 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:14:52,748 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:14:53,237 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:14:53,238 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:14:53,239 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:14:53,239 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:14:53,785 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:14:54,785 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:14:55,993 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:14:56,574 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:14:56,576 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:14:56,576 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:15:01,660 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:15:06,661 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:15:11,574 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:15:11,575 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:15:11,576 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:15:12,612 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:15:13,799 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:15:13,800 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:15:13,801 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:15:13,801 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:15:14,793 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:15:16,794 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:15:18,583 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:15:22,754 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:15:23,754 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:15:26,574 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:15:26,576 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:15:26,576 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:15:29,699 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:15:34,289 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:15:34,290 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:15:34,291 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:15:34,291 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:15:34,801 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:15:35,087 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:15:36,801 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:15:40,088 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:15:41,574 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:15:41,576 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:15:41,577 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:15:45,631 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:15:50,632 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:15:52,750 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:15:55,751 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:15:56,090 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:15:56,091 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:15:56,092 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:15:56,094 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:15:56,574 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:15:56,576 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:15:56,576 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:15:56,808 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:15:58,809 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:16:01,197 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:16:06,198 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:16:11,199 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:16:11,574 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:16:11,576 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:16:11,576 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:16:16,658 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:16:18,836 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:16:18,837 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:16:18,837 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:16:18,838 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:16:19,817 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:16:20,818 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:16:22,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:16:22,752 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:16:26,574 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:16:26,576 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:16:26,576 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:16:27,631 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:16:32,632 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:16:37,632 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:16:40,293 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:16:40,294 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:16:40,295 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:16:40,297 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:16:40,825 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:16:41,574 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:16:41,576 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:16:41,576 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:16:42,653 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:16:42,826 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:16:47,655 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:16:52,656 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:16:52,753 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:16:56,575 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:16:56,576 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:16:56,576 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:16:58,625 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:17:01,867 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:17:01,869 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:17:01,869 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:17:01,870 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:17:02,833 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:17:04,617 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:17:04,834 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:17:09,618 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:17:11,575 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:17:11,577 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:17:11,577 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:17:14,663 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:17:19,664 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:17:22,754 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:17:23,880 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:17:23,881 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:17:23,882 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:17:23,882 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:17:24,842 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:17:25,603 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:17:26,575 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:17:26,577 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:17:26,577 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:17:26,842 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:17:30,615 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:17:35,615 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:17:40,616 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:17:41,575 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:17:41,576 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:17:41,577 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:17:45,285 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:17:45,287 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:17:45,287 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:17:45,288 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:17:45,849 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:17:46,031 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:17:46,850 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:17:51,032 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:17:52,755 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:17:56,575 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:17:56,577 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:17:56,578 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:17:56,599 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:18:01,600 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:18:05,593 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:18:05,595 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:18:05,595 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:18:05,596 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:18:05,857 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:18:06,857 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:18:07,336 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:18:11,575 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:18:11,577 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:18:11,578 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:18:12,604 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:18:17,605 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:18:22,606 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:18:22,756 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:18:26,155 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:18:26,156 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:18:26,157 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:18:26,157 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:18:26,597 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:18:26,634 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:18:26,635 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:18:26,864 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:18:27,928 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:18:28,865 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:18:32,929 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:18:37,930 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:18:41,581 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:18:41,581 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:18:41,582 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:18:43,677 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:18:48,678 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:18:52,539 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:18:52,540 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:18:52,540 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:18:52,541 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:18:52,757 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:18:52,874 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:18:54,334 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:18:54,875 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:18:56,581 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:18:56,581 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:18:56,582 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:18:59,704 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:19:04,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:19:09,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:19:11,581 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:19:11,582 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:19:11,582 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:19:15,675 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:19:16,376 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:19:16,377 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:19:16,377 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:19:16,378 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:19:16,883 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:19:18,884 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:19:21,145 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:19:22,757 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:19:26,581 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:19:26,583 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:19:26,583 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:19:26,674 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:19:31,675 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:19:36,676 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:19:37,808 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:19:37,809 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:19:37,809 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:19:37,810 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:19:37,890 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:19:40,892 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:19:41,581 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:19:41,582 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:19:41,583 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:19:42,616 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:19:47,617 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:19:52,618 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:19:52,758 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:19:56,581 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:19:56,582 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:19:56,583 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:19:57,678 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:19:58,528 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:19:58,529 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:19:58,529 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:19:58,530 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:19:58,899 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:20:00,900 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:20:03,409 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:20:08,410 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:20:11,581 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:20:11,582 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:20:11,582 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:20:13,622 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:20:16,902 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:20:16,903 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:20:16,903 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:20:16,904 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:20:16,907 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:20:18,654 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:20:18,908 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:20:22,759 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:20:23,760 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:20:26,581 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:20:26,582 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:20:26,583 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:20:29,645 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:20:34,646 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:20:39,647 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:20:40,161 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:20:40,162 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:20:40,162 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:20:40,162 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:20:40,916 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:20:41,582 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:20:41,583 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:20:41,584 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:20:42,917 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:20:45,625 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:20:50,626 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:20:52,760 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:20:55,762 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:20:56,582 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:20:56,584 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:20:56,584 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:21:01,693 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:21:03,191 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:21:03,192 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:21:03,193 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:21:03,193 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:21:03,924 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:21:04,925 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:21:06,928 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:21:11,582 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:21:11,583 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:21:11,583 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:21:12,686 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:21:17,687 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:21:22,688 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:21:22,762 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:21:23,408 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:21:23,409 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:21:23,410 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:21:23,410 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:21:23,932 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:21:24,932 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:21:26,583 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:21:26,583 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:21:26,583 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:21:27,690 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:21:32,691 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:21:37,692 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:21:41,582 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:21:41,583 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:21:41,583 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:21:41,940 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:21:41,942 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:21:41,942 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:21:41,942 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:21:42,939 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:21:42,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:21:44,940 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:21:47,954 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:21:52,763 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:21:53,763 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:21:56,584 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:21:56,585 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:21:56,585 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:21:59,632 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:22:01,562 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:22:01,564 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:22:01,564 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:22:01,565 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:22:01,946 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:22:02,946 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:22:05,318 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:22:10,319 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:22:11,584 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:22:11,585 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:22:11,587 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:22:15,664 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:22:20,665 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:22:21,543 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:22:21,544 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:22:21,544 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:22:21,545 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:22:21,954 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:22:22,763 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:22:22,954 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:22:25,764 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:22:26,583 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:22:26,584 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:22:26,584 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:22:31,685 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:22:36,686 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:22:41,583 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:22:41,584 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:22:41,584 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:22:41,705 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:22:43,083 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:22:43,085 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:22:43,085 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:22:43,086 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:22:43,963 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:22:44,963 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:22:46,851 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:22:51,853 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:22:52,765 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:22:56,583 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:22:56,585 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:22:56,586 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:22:57,652 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:23:02,653 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:23:03,021 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:23:03,022 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:23:03,023 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:23:03,023 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:23:03,971 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:23:05,972 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:23:07,769 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:23:11,584 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:23:11,584 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:23:11,585 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:23:13,705 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:23:18,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:23:22,373 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:23:22,374 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:23:22,375 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:23:22,375 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:23:22,766 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:23:22,978 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:23:24,104 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:23:25,980 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:23:26,584 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:23:26,584 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:23:26,585 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:23:29,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:23:34,708 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:23:39,709 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:23:41,584 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:23:41,585 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:23:41,585 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:23:42,010 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:23:42,011 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:23:42,011 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:23:42,012 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:23:42,986 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:23:43,987 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:23:44,849 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:23:49,850 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:23:52,766 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:23:55,767 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:23:56,584 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:23:56,585 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:23:56,586 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:24:01,684 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:24:01,890 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:24:01,891 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:24:01,892 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:24:01,892 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:24:01,993 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:24:03,994 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:24:07,628 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:24:11,585 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:24:11,585 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:24:11,586 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:24:12,711 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:24:17,711 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:24:20,491 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:24:20,493 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:24:20,493 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:24:20,493 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:24:21,002 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:24:22,767 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:24:22,768 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:24:24,003 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:24:26,585 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:24:26,586 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:24:26,586 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:24:28,628 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:24:33,629 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:24:38,630 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:24:40,287 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:24:40,288 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:24:40,288 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:24:40,289 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:24:41,010 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:24:41,586 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:24:41,586 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:24:41,589 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:24:43,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:24:44,011 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:24:48,708 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:24:52,768 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:24:53,769 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:24:56,586 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:24:56,586 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:24:56,591 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:24:59,681 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:25:00,608 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:25:00,609 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:25:00,610 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:25:00,610 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:25:01,017 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:25:04,018 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:25:05,600 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:25:10,601 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:25:11,586 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:25:11,586 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:25:11,589 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:25:15,660 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:25:20,661 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:25:22,769 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:25:22,802 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:25:22,803 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:25:22,803 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:25:22,804 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:25:23,026 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:25:26,028 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:25:26,550 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:25:26,586 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:25:26,587 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:25:26,590 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:25:31,630 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:25:36,631 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:25:40,172 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:25:40,173 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:25:40,173 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:25:40,173 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:25:41,034 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:25:41,586 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:25:41,586 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:25:41,589 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:25:41,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:25:42,034 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:25:46,708 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:25:51,709 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:25:52,770 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:25:56,586 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:25:56,587 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:25:56,591 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:25:57,625 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:25:59,904 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:25:59,905 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:25:59,905 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:25:59,906 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:26:00,041 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:26:02,042 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:26:03,393 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:26:08,394 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:26:11,586 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:26:11,587 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:26:11,589 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:26:13,663 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:26:18,664 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:26:20,082 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:26:20,083 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:26:20,083 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:26:20,084 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:26:21,049 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:26:22,050 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:26:22,771 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:26:23,772 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:26:26,586 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:26:26,587 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:26:26,590 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:26:29,698 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:26:34,699 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:26:39,700 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:26:41,244 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:26:41,245 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:26:41,246 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:26:41,246 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:26:41,764 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:26:41,765 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:26:41,804 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:26:42,058 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:26:44,059 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:26:45,064 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:26:50,065 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:26:52,772 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:26:55,773 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:26:56,698 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:26:56,704 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:26:56,705 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:27:00,450 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:27:00,451 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:27:00,451 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:27:00,451 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:27:01,066 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:27:01,194 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:27:04,068 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:27:06,195 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:27:11,196 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:27:11,698 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:27:11,704 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:27:11,704 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:27:16,767 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:27:21,768 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:27:22,773 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:27:22,933 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:27:22,934 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:27:22,935 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:27:22,935 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:27:23,075 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:27:26,077 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:27:26,698 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:27:26,705 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:27:26,705 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:27:26,786 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:27:31,787 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:27:36,788 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:27:41,699 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:27:41,705 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:27:41,705 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:27:41,835 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:27:42,331 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:27:42,333 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:27:42,333 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:27:42,333 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:27:43,083 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:27:44,084 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:27:47,091 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:27:52,092 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:27:52,774 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:27:56,699 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:27:56,705 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:27:56,705 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:27:57,735 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:28:02,736 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:28:04,524 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:28:04,525 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:28:04,525 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:28:04,526 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:28:05,092 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:28:08,093 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:28:08,335 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:28:11,699 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:28:11,705 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:28:11,706 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:28:13,815 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:28:18,817 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:28:22,757 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:28:22,759 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:28:22,759 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:28:22,760 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:28:22,775 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:28:23,099 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:28:24,508 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:28:26,100 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:28:26,699 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:28:26,705 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:28:26,705 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:28:29,747 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:28:34,748 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:28:39,749 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:28:41,700 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:28:41,704 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:28:41,705 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:28:43,526 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:28:43,527 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:28:43,527 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:28:43,528 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:28:44,108 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:28:45,270 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:28:46,108 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:28:50,271 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:28:52,776 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:28:55,777 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:28:56,699 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:28:56,705 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:28:56,705 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:29:00,810 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:29:04,739 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:29:04,740 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:29:04,740 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:29:04,741 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:29:05,116 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:29:06,477 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:29:08,118 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:29:11,478 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:29:11,700 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:29:11,705 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:29:11,705 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:29:16,749 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:29:21,750 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:29:22,777 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:29:26,700 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:29:26,705 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:29:26,706 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:29:26,791 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:29:27,165 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:29:27,167 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:29:27,167 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:29:27,168 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:29:28,125 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:29:30,126 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:29:31,899 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:29:36,900 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:29:41,700 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:29:41,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:29:41,707 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:29:42,783 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:29:47,785 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:29:52,705 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:29:52,706 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:29:52,706 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:29:52,707 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:29:52,777 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:29:53,135 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:29:53,434 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:29:56,136 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:29:56,700 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:29:56,705 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:29:56,706 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:29:58,820 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:30:03,821 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:30:08,823 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:30:11,701 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:30:11,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:30:11,706 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:30:14,761 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:30:16,775 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:30:16,776 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:30:16,777 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:30:16,777 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:30:17,145 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:30:20,146 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:30:20,514 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:30:22,778 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:30:25,780 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:30:26,701 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:30:26,705 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:30:26,706 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:30:30,787 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:30:35,788 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:30:38,126 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:30:38,127 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:30:38,127 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:30:38,127 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:30:38,154 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:30:40,155 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:30:40,858 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:30:41,705 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:30:41,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:30:41,706 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:30:46,812 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:30:51,813 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:30:52,779 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:30:56,704 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:30:56,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:30:56,706 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:30:56,842 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:30:57,634 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:30:57,635 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:30:57,635 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:30:57,636 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:30:58,163 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:31:00,164 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:31:02,391 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:31:07,392 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:31:11,704 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:31:11,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:31:11,707 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:31:12,806 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:31:17,015 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:31:17,016 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:31:17,016 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:31:17,017 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:31:17,170 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:31:18,766 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:31:20,172 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:31:22,781 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:31:23,783 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:31:26,704 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:31:26,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:31:26,707 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:31:29,748 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:31:34,749 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:31:35,963 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:31:35,964 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:31:35,965 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:31:35,965 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:31:36,177 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:31:38,178 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:31:39,982 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:31:41,704 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:31:41,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:31:41,707 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:31:45,867 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:31:50,868 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:31:52,785 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:31:56,207 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:31:56,208 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:31:56,209 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:31:56,209 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:31:56,209 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:31:56,705 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:31:56,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:31:56,707 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:31:57,185 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:31:58,186 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:32:01,977 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:32:06,978 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:32:11,705 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:32:11,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:32:11,707 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:32:12,747 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:32:17,748 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:32:17,888 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:32:17,889 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:32:17,890 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:32:17,890 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:32:18,193 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:32:20,194 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:32:22,782 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:32:22,783 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:32:26,705 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:32:26,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:32:26,707 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:32:28,762 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:32:33,762 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:32:38,763 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:32:41,705 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:32:41,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:32:41,707 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:32:41,840 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:32:41,841 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:32:41,841 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:32:41,843 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:32:42,203 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:32:44,204 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:32:44,562 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:32:49,563 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:32:52,783 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:32:54,784 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:32:56,705 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:32:56,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:32:56,707 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:32:59,801 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:33:04,801 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:33:06,172 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:33:06,173 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:33:06,174 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:33:06,174 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:33:06,214 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:33:08,215 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:33:09,950 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:33:11,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:33:11,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:33:11,707 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:33:15,791 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:33:20,792 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:33:22,784 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:33:25,351 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:33:25,352 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:33:25,352 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:33:25,353 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:33:26,104 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:33:26,222 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:33:26,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:33:26,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:33:26,707 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:33:28,223 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:33:31,801 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:33:36,803 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:33:41,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:33:41,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:33:41,708 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:33:42,777 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:33:47,331 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:33:47,333 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:33:47,333 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:33:47,333 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:33:48,169 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:33:48,231 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:33:50,232 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:33:52,786 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:33:53,786 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:33:56,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:33:56,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:33:56,708 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:33:58,800 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:34:03,801 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:34:08,801 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:34:08,957 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:34:08,958 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:34:08,958 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:34:08,959 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:34:09,239 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:34:11,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:34:11,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:34:11,708 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:34:12,240 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:34:14,795 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:34:19,795 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:34:22,786 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:34:25,788 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:34:26,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:34:26,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:34:26,708 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:34:31,702 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:34:31,703 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:34:31,704 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:34:31,704 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:34:31,705 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:34:32,247 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:34:34,248 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:34:37,483 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:34:41,706 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:34:41,708 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:34:41,708 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:34:42,826 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:34:47,827 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:34:52,787 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:34:53,788 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:34:54,583 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:34:54,584 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:34:54,585 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:34:54,585 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:34:55,255 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:34:56,255 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:34:56,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:34:56,708 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:34:56,708 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:34:58,804 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:35:03,805 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:35:08,806 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:35:11,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:35:11,709 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:35:11,710 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:35:13,571 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:35:13,572 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:35:13,572 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:35:13,573 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:35:14,263 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:35:14,574 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:35:16,264 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:35:19,621 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:35:22,789 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:35:24,789 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:35:26,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:35:26,709 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:35:26,710 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:35:29,822 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:35:34,216 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:35:34,217 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:35:34,218 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:35:34,218 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:35:34,270 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:35:34,969 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:35:36,273 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:35:39,971 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:35:41,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:35:41,709 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:35:41,709 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:35:45,810 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:35:50,811 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:35:52,789 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:35:56,119 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:35:56,120 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:35:56,121 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:35:56,121 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:35:56,122 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:35:56,281 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:35:56,970 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:35:56,970 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:35:57,010 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:35:58,282 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:36:01,259 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:36:06,260 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:36:11,261 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:36:11,909 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:36:11,917 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:36:11,917 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:36:16,196 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:36:16,197 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:36:16,197 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:36:16,198 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:36:16,288 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:36:17,198 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:36:18,289 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:36:22,228 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:36:22,790 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:36:26,910 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:36:26,915 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:36:26,916 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:36:28,029 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:36:33,030 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:36:37,037 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:36:37,038 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:36:37,039 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:36:37,039 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:36:37,301 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:36:38,301 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:36:38,792 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:36:41,910 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:36:41,915 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:36:41,916 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:36:44,018 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:36:49,020 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:36:52,791 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:36:54,792 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:36:56,910 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:36:56,916 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:36:56,916 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:36:57,520 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:36:57,521 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:36:57,521 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:36:57,522 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:36:58,309 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:37:00,259 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:37:00,309 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:37:05,260 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:37:10,261 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:37:11,910 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:37:11,916 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:37:11,916 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:37:16,032 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:37:19,054 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:37:19,055 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:37:19,056 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:37:19,056 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:37:19,316 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:37:20,316 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:37:21,800 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:37:22,792 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:37:26,910 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:37:26,916 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:37:26,916 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:37:26,917 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:37:32,011 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:37:37,012 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:37:38,584 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:37:38,585 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:37:38,585 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:37:38,586 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:37:39,324 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:37:40,324 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:37:41,911 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:37:41,916 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:37:41,917 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:37:42,015 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:37:47,016 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:37:52,017 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:37:52,793 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:37:56,910 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:37:56,917 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:37:56,918 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:37:57,027 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:38:00,482 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:38:00,483 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:38:00,484 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:38:00,484 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:38:01,333 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:38:02,207 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:38:02,333 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:38:07,208 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:38:11,911 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:38:11,916 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:38:11,917 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:38:12,988 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:38:17,989 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:38:18,949 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:38:18,950 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:38:18,950 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:38:18,950 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:38:19,340 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:38:20,340 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:38:22,794 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:38:23,795 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:38:26,911 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:38:26,917 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:38:26,918 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:38:28,999 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:38:34,000 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:38:37,176 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:38:37,178 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:38:37,178 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:38:37,178 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:38:37,348 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:38:38,349 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:38:39,981 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:38:41,911 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:38:41,917 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:38:41,917 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:38:45,039 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:38:50,040 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:38:52,795 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:38:55,796 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:38:56,764 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:38:56,765 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:38:56,766 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:38:56,766 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:38:56,911 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:38:56,918 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:38:56,918 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:38:57,355 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:38:58,356 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:39:01,523 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:39:06,524 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:39:11,525 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:39:11,911 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:39:11,917 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:39:11,917 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:39:16,958 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:39:18,577 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:39:18,579 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:39:18,579 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:39:18,579 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:39:19,363 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:39:20,364 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:39:22,327 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:39:22,796 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:39:26,911 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:39:26,917 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:39:26,918 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:39:28,048 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:39:33,049 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:39:38,050 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:39:40,609 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:39:40,611 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:39:40,611 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:39:40,611 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:39:41,373 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:39:41,912 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:39:41,918 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:39:41,918 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:39:42,373 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:39:43,055 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:39:48,056 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:39:52,801 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:39:53,802 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:39:56,912 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:39:56,919 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:39:56,920 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:39:59,016 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:39:59,716 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:39:59,717 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:39:59,718 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:39:59,718 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:40:00,380 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:40:02,381 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:40:04,799 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:40:09,800 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:40:11,912 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:40:11,918 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:40:11,919 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:40:14,959 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:40:19,597 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:40:19,598 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:40:19,599 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:40:19,599 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:40:20,348 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:40:20,387 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:40:22,388 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:40:22,798 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:40:25,799 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:40:26,912 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:40:26,918 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:40:26,919 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:40:30,969 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:40:35,969 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:40:40,971 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:40:41,912 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:40:41,918 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:40:41,918 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:40:44,877 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:40:44,878 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:40:44,879 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:40:44,881 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:40:45,397 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:40:46,397 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:40:46,649 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:40:51,650 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:40:52,798 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:40:56,800 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:40:56,912 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:40:56,918 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:40:56,919 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:41:02,006 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:41:04,893 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:41:04,895 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:41:04,895 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:41:04,895 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:41:05,404 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:41:06,404 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:41:07,637 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:41:11,913 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:41:11,919 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:41:11,919 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:41:13,025 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:41:18,026 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:41:22,799 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:41:23,801 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:41:24,526 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:41:24,527 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:41:24,528 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:41:24,528 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:41:25,412 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:41:26,412 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:41:26,913 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:41:26,919 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:41:26,919 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:41:28,967 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:41:33,968 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:41:38,969 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:41:41,913 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:41:41,919 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:41:41,919 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:41:44,044 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:41:44,579 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:41:44,581 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:41:44,581 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:41:44,581 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:41:45,419 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:41:46,419 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:41:49,325 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:41:52,800 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:41:54,801 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:41:56,914 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:41:56,919 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:41:56,919 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:42:00,048 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:42:05,049 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:42:07,490 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:42:07,492 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:42:07,492 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:42:07,492 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:42:08,428 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:42:10,279 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:42:10,429 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:42:11,913 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:42:11,920 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:42:11,920 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:42:16,050 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:42:21,051 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:42:22,801 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:42:26,802 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:42:26,913 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:42:26,919 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:42:26,920 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:42:28,958 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:42:28,959 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:42:28,959 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:42:28,960 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:42:29,436 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:42:30,437 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:42:32,696 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:42:37,697 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:42:41,914 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:42:41,919 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:42:41,920 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:42:43,065 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:42:48,066 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:42:50,436 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:42:50,437 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:42:50,437 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:42:50,438 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:42:50,444 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:42:52,445 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:42:52,802 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:42:53,803 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:42:56,914 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:42:56,919 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:42:56,920 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:42:59,049 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:43:04,050 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:43:09,051 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:43:10,390 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:43:10,391 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:43:10,392 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:43:10,392 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:43:10,452 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:43:11,914 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:43:11,919 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:43:11,920 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:43:12,453 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:43:15,041 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:43:20,042 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:43:22,803 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:43:25,804 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:43:26,914 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:43:26,920 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:43:26,920 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:43:31,027 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:43:31,626 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:43:31,627 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:43:31,628 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:43:31,628 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:43:32,461 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:43:34,462 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:43:36,414 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:43:41,415 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:43:41,914 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:43:41,920 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:43:41,920 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:43:46,996 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:43:51,997 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:43:52,804 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:43:56,914 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:43:56,920 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:43:56,920 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:43:57,009 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:43:57,222 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:43:57,223 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:43:57,223 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:43:57,224 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:43:57,472 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:44:00,473 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:44:02,974 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:44:07,975 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:44:11,914 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:44:11,920 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:44:11,920 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:44:13,022 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:44:17,563 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:44:17,564 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:44:17,564 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:44:17,565 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:44:18,327 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:44:18,480 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:44:20,481 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:44:22,806 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:44:23,806 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:44:26,914 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:44:26,920 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:44:26,920 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:44:28,955 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:44:33,956 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:44:38,957 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:44:40,232 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:44:40,233 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:44:40,233 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:44:40,234 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:44:40,488 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:44:41,915 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:44:41,920 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:44:41,921 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:44:42,489 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:44:44,037 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:44:49,038 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:44:52,806 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:44:54,808 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:44:56,915 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:44:56,920 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:44:56,921 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:45:00,004 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:45:04,154 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:45:04,155 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:45:04,156 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:45:04,156 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:45:04,497 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:45:05,925 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:45:06,498 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:45:10,926 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:45:11,915 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:45:11,921 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:45:11,921 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:45:16,016 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:45:21,017 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:45:22,807 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:45:26,809 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:45:26,851 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:45:26,852 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:45:26,852 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:45:26,852 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:45:26,915 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:45:26,920 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:45:26,921 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:45:27,506 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:45:28,506 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:45:32,608 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:45:37,609 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:45:41,916 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:45:41,921 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:45:41,922 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:45:43,021 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:45:48,022 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:45:49,673 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:45:49,674 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:45:49,674 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:45:49,675 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:45:50,516 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:45:52,809 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:45:53,517 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:45:53,809 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:45:56,915 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:45:56,921 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:45:56,921 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:45:58,996 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:46:03,997 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:46:08,998 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:46:11,180 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:46:11,181 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:46:11,181 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:46:11,182 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:46:11,524 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:46:11,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:46:11,944 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:46:11,945 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:46:14,081 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:46:15,525 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:46:19,082 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:46:22,809 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:46:24,810 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:46:26,939 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:46:26,940 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:46:26,940 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:46:30,015 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:46:32,590 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:46:32,591 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:46:32,591 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:46:32,592 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:46:33,532 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:46:35,349 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:46:35,533 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:46:40,350 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:46:41,939 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:46:41,940 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:46:41,940 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:46:45,966 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:46:50,967 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:46:51,768 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:46:51,770 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:46:51,770 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:46:51,771 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:46:52,539 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:46:52,810 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:46:55,541 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:46:56,811 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:46:56,940 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:46:56,940 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:46:56,941 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:47:02,062 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:47:07,063 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:47:11,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:47:11,944 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:47:11,944 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:47:11,985 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:47:12,015 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:47:12,015 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:47:12,016 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:47:12,547 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:47:12,699 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:47:15,548 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:47:17,700 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:47:22,701 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:47:22,811 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:47:26,940 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:47:26,940 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:47:26,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:47:28,067 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:47:33,069 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:47:34,999 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:47:35,000 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:47:35,001 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:47:35,001 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:47:35,557 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:47:37,557 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:47:38,739 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:47:41,940 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:47:41,940 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:47:41,944 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:47:43,990 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:47:48,992 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:47:52,812 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:47:54,813 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:47:55,399 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:47:55,400 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:47:55,401 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:47:55,401 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:47:55,564 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:47:56,940 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:47:56,940 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:47:56,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:47:57,565 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:48:00,018 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:48:05,019 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:48:10,020 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:48:11,940 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:48:11,940 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:48:11,981 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:48:15,051 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:48:15,383 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:48:15,385 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:48:15,385 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:48:15,385 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:48:15,572 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:48:19,575 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:48:20,291 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:48:22,813 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:48:25,814 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:48:26,940 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:48:26,941 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:48:26,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:48:31,020 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:48:36,021 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:48:36,502 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:48:36,504 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:48:36,504 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:48:36,504 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:48:36,582 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:48:39,584 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:48:41,268 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:48:41,940 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:48:41,941 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:48:41,944 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:48:47,046 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:48:52,046 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:48:52,814 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:48:56,940 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:48:56,941 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:48:56,981 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:48:57,212 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:48:57,213 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:48:57,214 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:48:57,214 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:48:57,214 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:48:57,590 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:48:59,591 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:49:02,949 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:49:07,950 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:49:11,941 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:49:11,941 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:49:11,945 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:49:12,994 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:49:17,996 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:49:18,805 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:49:18,806 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:49:18,806 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:49:18,807 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:49:19,599 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:49:21,600 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:49:22,815 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:49:23,816 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:49:26,941 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:49:26,941 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:49:26,944 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:49:29,063 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:49:34,063 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:49:37,662 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:49:37,664 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:49:37,664 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:49:37,664 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:49:38,606 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:49:39,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:49:41,607 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:49:41,941 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:49:41,942 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:49:41,944 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:49:45,060 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:49:50,061 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:49:52,816 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:49:55,817 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:49:56,941 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:49:56,941 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:49:56,982 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:49:58,223 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:49:58,224 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:49:58,224 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:49:58,224 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:49:58,614 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:50:00,998 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:50:01,615 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:50:05,999 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:50:11,000 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:50:11,941 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:50:11,942 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:50:11,945 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:50:16,078 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:50:18,085 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:50:18,086 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:50:18,086 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:50:18,086 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:50:18,622 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:50:21,623 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:50:21,841 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:50:22,817 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:50:26,941 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:50:26,942 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:50:26,943 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:50:27,039 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:50:32,039 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:50:37,041 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:50:39,514 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:50:39,515 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:50:39,515 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:50:39,516 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:50:39,631 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:50:41,632 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:50:41,941 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:50:41,942 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:50:41,943 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:50:42,985 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:50:47,986 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:50:52,818 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:50:53,819 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:50:56,941 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:50:56,942 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:50:56,983 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:50:59,022 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:50:59,355 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:50:59,356 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:50:59,356 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:50:59,357 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:50:59,638 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:51:01,639 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:51:04,107 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:51:09,108 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:51:11,942 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:51:11,942 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:51:11,944 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:51:14,996 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:51:19,079 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:51:19,081 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:51:19,081 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:51:19,081 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:51:19,647 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:51:20,824 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:51:21,648 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:51:22,818 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:51:26,819 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:51:26,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:51:26,943 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:51:26,946 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:51:31,969 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:51:36,970 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:51:40,358 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:51:40,359 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:51:40,359 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:51:40,359 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:51:40,655 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:51:41,942 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:51:41,942 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:51:41,945 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:51:42,051 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:51:43,656 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:51:47,052 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:51:52,053 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:51:52,819 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:51:56,942 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:51:56,942 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:51:56,983 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:51:57,057 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:52:01,490 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:52:01,492 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:52:01,492 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:52:01,493 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:52:01,664 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:52:02,329 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:52:03,666 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:52:07,330 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:52:11,942 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:52:11,943 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:52:11,946 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:52:13,042 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:52:18,043 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:52:22,820 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:52:23,821 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:52:25,580 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:52:25,582 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:52:25,582 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:52:25,583 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:52:25,674 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:52:26,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:52:26,943 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:52:26,948 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:52:27,675 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:52:29,072 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:52:34,073 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:52:39,074 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:52:41,942 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:52:41,943 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:52:41,984 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:52:45,024 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:52:47,077 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:52:47,078 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:52:47,078 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:52:47,079 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:52:47,683 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:52:49,684 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:52:50,075 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:52:52,821 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:52:55,822 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:52:56,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:52:56,943 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:52:56,946 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:53:01,060 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:53:06,061 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:53:11,062 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:53:11,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:53:11,943 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:53:11,984 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:53:12,590 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:53:12,591 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:53:12,592 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:53:12,592 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:53:12,692 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:53:15,694 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:53:16,466 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:53:21,467 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:53:22,821 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:53:26,823 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:53:26,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:53:26,943 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:53:26,946 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:53:31,993 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:53:33,595 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:53:33,596 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:53:33,596 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:53:33,596 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:53:33,700 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:53:35,701 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:53:37,331 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:53:41,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:53:41,944 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:53:41,946 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:53:43,077 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:53:48,078 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:53:52,822 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:53:53,823 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:53:54,855 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:53:54,857 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:53:54,857 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:53:54,858 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:53:55,709 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:53:56,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:53:56,944 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:53:56,947 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:53:57,710 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:53:59,016 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:54:04,017 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:54:09,018 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:54:11,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:54:11,944 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:54:11,984 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:54:14,990 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:54:16,876 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:54:16,877 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:54:16,878 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:54:16,878 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:54:17,716 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:54:19,717 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:54:20,721 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:54:22,823 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:54:25,824 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:54:26,944 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:54:26,944 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:54:26,946 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:54:31,047 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:54:36,048 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:54:37,285 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:54:37,286 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:54:37,287 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:54:37,287 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:54:37,724 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:54:39,724 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:54:41,944 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:54:41,944 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:54:41,946 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:54:41,981 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:54:46,982 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:54:51,983 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:54:52,824 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:54:56,944 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:54:56,944 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:54:56,982 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:54:57,983 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:54:59,460 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:54:59,461 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:54:59,462 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:54:59,462 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:54:59,732 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:55:01,733 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:55:03,200 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:55:08,201 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:55:11,944 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:55:11,944 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:55:11,947 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:55:14,072 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:55:19,074 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:55:19,817 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:55:19,819 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:55:19,819 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:55:19,819 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:55:20,741 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:55:21,742 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:55:22,825 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:55:24,826 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:55:26,944 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:55:26,945 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:55:26,949 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:55:29,979 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:55:34,981 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:55:39,981 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:55:41,465 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:55:41,466 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:55:41,467 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:55:41,467 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:55:41,750 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:55:41,966 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:55:41,966 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:55:41,967 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:55:43,751 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:55:45,245 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:55:50,246 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:55:52,826 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:55:55,827 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:55:56,950 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:55:56,951 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:55:56,964 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:56:01,028 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:56:04,811 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:56:04,812 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:56:04,813 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:56:04,813 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:56:05,760 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:56:06,815 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:56:07,761 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:56:11,816 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:56:11,950 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:56:11,950 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:56:11,964 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:56:17,074 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:56:22,075 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:56:22,827 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:56:26,950 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:56:26,951 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:56:26,991 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:56:27,211 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:56:27,218 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:56:27,219 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:56:27,219 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:56:27,219 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:56:27,768 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:56:29,769 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:56:33,095 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:56:38,096 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:56:41,950 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:56:41,951 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:56:41,964 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:56:43,979 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:56:47,527 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:56:47,529 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:56:47,529 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:56:47,529 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:56:47,775 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:56:49,263 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:56:49,776 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:56:52,828 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:56:54,829 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:56:56,950 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:56:56,951 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:56:56,964 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:56:59,990 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:57:04,991 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:57:07,795 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:57:07,796 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:57:07,796 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:57:07,797 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:57:08,783 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:57:09,784 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:57:10,550 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:57:11,951 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:57:11,951 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:57:11,965 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:57:16,021 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:57:21,022 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:57:22,829 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:57:26,830 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:57:26,951 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:57:26,951 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:57:26,992 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:57:29,622 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:57:29,624 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:57:29,624 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:57:29,624 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:57:29,790 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:57:31,791 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:57:32,366 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:57:37,367 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:57:41,951 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:57:41,951 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:57:41,965 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:57:43,015 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:57:48,015 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:57:49,918 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:57:49,919 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:57:49,919 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:57:49,920 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:57:50,798 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:57:51,798 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:57:52,830 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:57:53,831 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:57:56,951 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:57:56,952 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:57:56,965 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:57:58,974 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:58:03,975 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:58:08,976 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:58:11,713 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:58:11,714 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:58:11,714 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:58:11,714 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:58:11,806 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:58:12,214 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:58:12,215 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:58:12,215 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:58:13,807 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:58:14,495 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:58:19,496 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:58:22,831 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:58:24,832 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:58:27,152 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:58:27,157 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:58:27,157 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:58:30,185 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:58:31,630 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:58:31,632 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:58:31,632 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:58:31,632 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:58:31,814 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:58:33,815 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:58:35,387 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:58:40,388 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:58:42,153 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:58:42,156 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:58:42,157 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:58:46,203 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:58:51,203 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:58:52,314 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:58:52,315 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:58:52,315 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:58:52,315 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:58:52,823 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:58:52,831 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:58:55,824 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:58:57,065 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:58:57,152 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:58:57,156 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:58:57,157 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:59:02,208 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:59:07,209 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:59:12,152 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:59:12,156 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:59:12,157 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:59:13,196 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:59:15,219 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:59:15,221 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:59:15,221 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:59:15,221 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:59:15,832 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:59:17,833 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:59:19,012 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:59:22,832 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:59:24,833 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:59:27,152 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:59:27,156 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:59:27,157 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:59:30,289 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:59:35,290 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:59:39,730 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:59:39,732 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:59:39,732 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:59:39,732 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:59:39,842 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 15:59:40,475 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:59:41,844 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 15:59:42,152 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:59:42,156 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:59:42,157 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:59:46,192 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:59:51,193 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:59:52,833 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 15:59:56,834 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 15:59:57,154 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 15:59:57,157 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 15:59:57,157 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 15:59:59,725 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 15:59:59,726 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 15:59:59,726 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 15:59:59,727 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 15:59:59,850 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:00:01,851 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:00:02,468 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:00:07,469 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:00:12,154 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:00:12,157 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:00:12,157 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:00:13,213 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:00:18,214 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:00:18,918 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:00:18,920 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:00:18,920 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:00:18,920 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:00:19,857 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:00:21,858 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:00:22,834 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:00:23,835 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:00:27,153 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:00:27,157 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:00:27,157 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:00:29,221 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:00:34,223 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:00:39,224 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:00:40,265 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:00:40,266 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:00:40,267 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:00:40,267 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:00:40,866 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:00:41,867 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:00:42,153 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:00:42,157 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:00:42,157 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:00:45,220 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:00:50,221 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:00:52,836 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:00:55,837 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:00:57,154 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:00:57,157 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:00:57,158 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:01:00,344 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:01:00,345 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:01:00,345 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:01:00,346 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:01:00,874 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:01:01,144 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:01:01,875 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:01:06,145 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:01:11,146 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:01:12,153 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:01:12,157 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:01:12,158 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:01:16,201 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:01:21,202 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:01:22,103 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:01:22,104 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:01:22,105 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:01:22,105 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:01:22,837 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:01:22,883 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:01:23,883 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:01:26,983 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:01:27,153 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:01:27,157 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:01:27,158 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:01:32,199 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:01:37,200 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:01:42,153 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:01:42,157 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:01:42,158 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:01:42,273 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:01:44,126 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:01:44,127 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:01:44,128 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:01:44,128 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:01:44,890 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:01:45,891 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:01:47,875 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:01:52,838 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:01:53,839 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:01:57,154 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:01:57,157 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:01:57,158 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:01:59,229 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:02:03,227 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:02:03,228 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:02:03,229 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:02:03,229 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:02:03,898 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:02:05,013 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:02:05,899 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:02:10,014 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:02:12,154 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:02:12,158 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:02:12,158 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:02:15,224 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:02:20,225 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:02:22,839 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:02:23,142 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:02:23,144 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:02:23,144 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:02:23,144 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:02:23,906 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:02:25,906 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:02:25,917 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:02:27,154 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:02:27,159 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:02:27,159 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:02:31,297 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:02:36,298 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:02:41,299 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:02:42,154 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:02:42,158 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:02:42,158 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:02:46,280 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:02:46,281 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:02:46,281 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:02:46,282 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:02:46,915 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:02:47,053 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:02:47,916 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:02:52,054 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:02:52,840 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:02:57,154 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:02:57,155 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:02:57,158 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:02:57,158 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:03:02,287 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:03:07,288 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:03:09,856 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:03:09,857 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:03:09,857 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:03:09,858 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:03:09,924 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:03:11,925 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:03:12,154 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:03:12,158 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:03:12,158 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:03:13,230 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:03:18,231 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:03:22,840 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:03:23,841 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:03:27,156 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:03:27,159 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:03:27,160 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:03:29,281 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:03:31,512 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:03:31,514 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:03:31,514 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:03:31,514 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:03:31,933 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:03:33,934 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:03:35,268 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:03:40,269 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:03:42,155 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:03:42,158 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:03:42,159 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:03:46,263 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:03:51,263 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:03:52,478 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:03:52,479 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:03:52,479 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:03:52,480 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:03:52,841 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:03:52,941 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:03:55,942 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:03:56,427 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:03:57,155 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:03:57,159 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:03:57,159 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:04:02,187 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:04:07,188 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:04:12,157 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:04:12,159 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:04:12,159 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:04:12,277 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:04:12,878 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:04:12,879 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:04:12,880 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:04:12,880 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:04:12,949 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:04:15,950 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:04:17,633 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:04:22,634 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:04:22,843 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:04:27,156 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:04:27,159 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:04:27,159 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:04:28,187 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:04:33,188 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:04:34,697 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:04:34,698 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:04:34,699 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:04:34,699 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:04:34,957 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:04:37,958 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:04:38,934 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:04:42,157 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:04:42,159 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:04:42,159 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:04:44,285 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:04:49,286 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:04:52,843 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:04:54,844 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:04:57,155 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:04:57,159 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:04:57,159 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:04:57,270 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:04:57,271 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:04:57,272 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:04:57,272 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:04:57,965 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:04:59,966 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:05:00,008 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:05:05,009 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:05:10,010 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:05:12,156 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:05:12,159 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:05:12,159 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:05:15,241 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:05:18,637 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:05:18,638 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:05:18,639 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:05:18,639 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:05:18,974 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:05:19,975 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:05:20,497 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:05:22,844 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:05:25,845 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:05:27,156 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:05:27,159 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:05:27,159 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:05:31,279 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:05:36,280 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:05:37,357 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:05:37,358 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:05:37,358 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:05:37,358 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:05:37,981 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:05:39,981 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:05:42,094 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:05:42,157 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:05:42,159 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:05:42,160 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:05:47,207 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:05:52,208 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:05:52,845 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:05:57,156 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:05:57,159 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:05:57,160 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:05:57,294 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:06:01,533 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:06:01,535 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:06:01,535 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:06:01,536 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:06:01,990 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:06:02,296 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:06:03,991 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:06:07,296 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:06:12,156 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:06:12,159 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:06:12,160 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:06:13,209 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:06:18,210 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:06:22,846 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:06:23,485 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:06:23,486 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:06:23,486 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:06:23,487 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:06:23,487 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:06:23,998 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:06:25,999 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:06:27,156 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:06:27,160 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:06:27,160 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:06:29,226 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:06:34,227 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:06:39,228 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:06:42,156 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:06:42,159 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:06:42,160 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:06:43,389 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:06:43,390 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:06:43,391 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:06:43,391 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:06:44,007 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:06:45,174 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:06:46,007 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:06:50,175 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:06:52,847 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:06:55,848 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:06:57,157 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:06:57,160 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:06:57,160 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:07:01,213 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:07:04,243 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:07:04,244 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:07:04,244 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:07:04,245 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:07:05,014 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:07:06,015 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:07:06,999 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:07:12,000 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:07:12,157 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:07:12,160 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:07:12,160 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:07:17,208 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:07:22,209 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:07:22,848 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:07:23,866 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:07:23,867 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:07:23,868 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:07:23,868 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:07:24,021 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:07:26,022 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:07:27,159 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:07:27,161 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:07:27,161 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:07:27,215 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:07:32,216 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:07:37,217 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:07:42,157 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:07:42,160 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:07:42,160 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:07:43,203 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:07:44,089 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:07:44,090 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:07:44,090 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:07:44,091 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:07:45,029 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:07:46,029 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:07:48,827 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:07:52,848 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:07:53,849 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:07:57,158 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:07:57,161 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:07:57,161 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:07:59,231 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:08:04,231 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:08:08,144 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:08:08,146 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:08:08,146 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:08:08,147 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:08:09,038 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:08:09,892 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:08:10,039 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:08:12,158 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:08:12,160 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:08:12,161 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:08:15,224 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:08:20,224 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:08:22,849 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:08:25,850 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:08:27,158 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:08:27,160 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:08:27,161 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:08:29,391 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:08:29,392 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:08:29,393 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:08:29,393 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:08:30,046 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:08:31,482 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:08:33,047 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:08:36,483 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:08:41,484 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:08:42,158 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:08:42,160 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:08:42,161 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:08:47,209 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:08:52,011 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:08:52,012 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:08:52,013 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:08:52,013 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:08:52,054 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:08:52,790 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:08:52,850 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:08:55,055 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:08:57,158 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:08:57,161 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:08:57,161 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:08:58,241 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:09:03,242 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:09:08,243 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:09:10,332 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:09:10,333 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:09:10,333 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:09:10,334 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:09:11,061 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:09:12,158 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:09:12,161 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:09:12,161 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:09:13,062 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:09:13,298 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:09:18,299 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:09:22,851 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:09:23,852 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:09:27,158 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:09:27,161 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:09:27,161 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:09:29,260 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:09:31,544 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:09:31,545 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:09:31,546 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:09:31,546 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:09:32,069 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:09:34,318 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:09:35,070 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:09:39,318 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:09:42,158 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:09:42,161 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:09:42,161 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:09:45,256 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:09:50,257 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:09:51,460 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:09:51,461 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:09:51,461 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:09:51,462 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:09:52,076 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:09:52,852 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:09:55,078 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:09:55,853 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:09:57,158 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:09:57,161 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:09:57,162 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:10:01,283 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:10:06,284 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:10:10,514 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:10:10,515 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:10:10,516 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:10:10,516 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:10:11,084 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:10:12,162 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:10:12,163 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:10:12,165 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:10:12,165 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:10:13,085 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:10:17,302 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:10:22,303 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:10:22,853 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:10:27,159 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:10:27,161 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:10:27,162 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:10:28,200 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:10:31,109 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:10:31,110 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:10:31,111 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:10:31,111 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:10:32,092 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:10:33,093 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:10:33,885 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:10:38,886 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:10:42,159 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:10:42,161 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:10:42,162 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:10:44,289 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:10:49,290 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:10:52,524 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:10:52,525 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:10:52,526 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:10:52,526 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:10:52,854 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:10:53,100 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:10:54,544 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:10:55,101 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:10:57,159 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:10:57,161 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:10:57,162 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:11:00,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:11:05,279 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:11:10,280 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:11:11,852 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:11:11,854 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:11:11,854 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:11:11,854 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:11:12,108 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:11:12,322 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:11:12,322 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:11:12,325 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:11:15,109 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:11:15,613 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:11:20,614 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:11:22,854 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:11:25,855 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:11:27,258 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:11:27,258 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:11:27,261 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:11:31,359 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:11:32,058 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:11:32,060 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:11:32,060 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:11:32,060 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:11:32,114 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:11:35,116 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:11:36,817 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:11:41,818 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:11:42,258 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:11:42,258 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:11:42,261 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:11:47,389 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:11:50,467 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:11:50,468 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:11:50,469 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:11:50,469 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:11:51,122 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:11:52,856 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:11:52,856 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:11:53,122 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:11:57,258 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:11:57,258 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:11:57,260 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:11:58,367 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:12:03,368 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:12:08,369 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:12:10,648 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:12:10,649 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:12:10,649 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:12:10,650 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:12:11,130 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:12:12,258 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:12:12,259 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:12:12,261 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:12:13,131 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:12:14,369 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:12:19,370 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:12:22,856 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:12:24,857 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:12:27,258 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:12:27,259 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:12:27,261 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:12:30,288 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:12:30,543 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:12:30,544 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:12:30,545 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:12:30,545 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:12:31,138 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:12:33,139 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:12:35,436 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:12:40,437 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:12:42,258 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:12:42,259 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:12:42,261 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:12:46,294 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:12:50,601 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:12:50,602 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:12:50,603 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:12:50,603 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:12:51,146 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:12:51,391 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:12:52,861 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:12:53,147 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:12:56,862 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:12:57,260 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:12:57,261 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:12:57,261 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:13:02,289 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:13:07,290 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:13:11,808 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:13:11,809 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:13:11,810 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:13:11,810 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:13:12,154 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:13:12,292 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:13:12,293 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:13:12,293 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:13:12,349 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:13:15,156 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:13:17,579 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:13:22,580 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:13:22,858 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:13:27,265 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:13:27,275 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:13:27,275 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:13:28,335 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:13:31,090 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:13:31,091 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:13:31,092 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:13:31,094 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:13:31,162 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:13:33,162 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:13:33,853 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:13:38,854 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:13:42,265 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:13:42,275 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:13:42,276 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:13:44,304 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:13:49,305 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:13:50,983 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:13:50,985 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:13:50,985 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:13:50,985 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:13:51,169 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:13:52,859 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:13:53,170 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:13:54,860 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:13:57,265 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:13:57,275 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:13:57,276 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:14:00,409 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:14:05,410 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:14:10,410 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:14:12,265 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:14:12,275 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:14:12,276 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:14:13,107 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:14:13,108 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:14:13,108 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:14:13,110 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:14:13,178 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:14:16,192 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:14:17,180 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:14:21,193 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:14:22,860 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:14:26,861 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:14:27,266 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:14:27,276 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:14:27,276 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:14:32,389 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:14:34,758 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:14:34,760 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:14:34,760 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:14:34,760 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:14:35,186 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:14:37,190 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:14:37,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:14:42,266 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:14:42,276 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:14:42,276 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:14:43,402 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:14:48,403 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:14:52,861 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:14:53,862 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:14:55,028 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:14:55,030 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:14:55,030 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:14:55,030 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:14:55,198 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:14:57,199 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:14:57,266 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:14:57,276 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:14:57,276 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:14:59,346 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:15:04,347 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:15:09,348 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:15:12,266 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:15:12,276 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:15:12,277 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:15:15,345 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:15:17,741 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:15:17,742 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:15:17,742 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:15:17,742 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:15:18,206 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:15:20,885 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:15:21,208 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:15:22,862 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:15:26,863 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:15:27,272 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:15:27,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:15:27,279 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:15:32,394 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:15:37,395 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:15:37,822 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:15:37,823 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:15:37,824 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:15:37,824 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:15:38,215 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:15:41,216 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:15:42,271 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:15:42,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:15:42,278 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:15:43,358 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:15:48,359 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:15:52,862 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:15:53,863 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:15:57,272 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:15:57,279 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:15:57,279 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:15:59,249 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:15:59,250 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:15:59,251 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:15:59,251 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:15:59,251 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:16:00,224 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:16:01,225 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:16:05,003 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:16:10,004 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:16:12,272 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:16:12,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:16:12,278 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:16:15,401 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:16:20,038 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:16:20,039 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:16:20,040 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:16:20,040 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:16:20,232 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:16:20,964 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:16:22,863 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:16:23,233 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:16:26,864 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:16:27,272 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:16:27,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:16:27,279 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:16:32,363 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:16:37,364 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:16:40,162 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:16:40,163 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:16:40,163 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:16:40,164 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:16:40,240 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:16:42,272 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:16:42,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:16:42,278 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:16:43,242 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:16:43,317 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:16:48,318 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:16:52,864 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:16:53,865 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:16:57,272 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:16:57,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:16:57,279 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:16:59,351 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:17:03,020 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:17:03,022 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:17:03,022 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:17:03,022 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:17:03,249 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:17:04,795 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:17:05,250 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:17:09,796 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:17:12,272 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:17:12,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:17:12,279 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:17:15,356 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:17:20,356 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:17:22,865 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:17:25,854 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:17:25,855 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:17:25,856 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:17:25,856 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:17:25,857 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:17:26,259 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:17:27,273 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:17:27,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:17:27,279 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:17:29,260 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:17:31,386 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:17:36,387 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:17:41,388 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:17:42,273 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:17:42,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:17:42,279 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:17:46,399 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:17:46,400 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:17:46,400 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:17:46,400 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:17:46,401 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:17:47,267 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:17:49,268 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:17:52,241 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:17:52,865 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:17:57,273 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:17:57,279 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:17:57,279 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:17:57,365 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:18:02,366 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:18:05,308 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:18:05,309 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:18:05,310 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:18:05,310 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:18:06,274 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:18:07,274 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:18:08,249 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:18:12,273 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:18:12,279 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:18:12,279 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:18:13,359 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:18:18,360 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:18:22,867 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:18:23,868 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:18:25,558 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:18:25,559 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:18:25,560 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:18:25,560 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:18:26,281 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:18:27,273 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:18:27,279 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:18:27,279 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:18:29,285 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:18:29,347 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:18:34,348 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:18:39,348 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:18:42,274 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:18:42,279 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:18:42,280 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:18:44,364 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:18:44,851 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:18:44,852 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:18:44,853 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:18:44,853 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:18:45,290 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:18:47,291 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:18:49,686 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:18:52,867 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:18:54,869 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:18:57,274 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:18:57,279 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:18:57,279 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:19:00,313 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:19:05,314 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:19:08,029 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:19:08,030 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:19:08,030 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:19:08,031 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:19:08,298 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:19:10,805 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:19:11,300 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:19:12,274 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:19:12,279 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:19:12,279 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:19:16,330 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:19:21,331 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:19:22,868 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:19:26,869 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:19:27,274 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:19:27,279 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:19:27,280 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:19:29,695 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:19:29,697 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:19:29,697 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:19:29,698 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:19:30,308 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:19:31,308 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:19:32,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:19:37,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:19:42,274 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:19:42,279 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:19:42,280 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:19:43,383 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:19:48,384 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:19:49,608 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:19:49,609 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:19:49,609 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:19:49,610 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:19:50,317 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:19:51,317 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:19:52,869 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:19:53,870 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:19:57,274 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:19:57,280 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:19:57,280 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:19:59,331 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:20:04,332 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:20:09,333 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:20:12,275 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:20:12,280 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:20:12,280 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:20:13,720 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:20:13,721 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:20:13,721 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:20:13,722 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:20:14,326 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:20:14,507 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:20:15,326 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:20:19,508 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:20:22,870 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:20:24,871 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:20:27,275 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:20:27,280 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:20:27,280 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:20:30,371 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:20:33,489 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:20:33,491 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:20:33,491 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:20:33,491 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:20:34,334 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:20:35,334 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:20:36,329 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:20:41,330 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:20:42,275 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:20:42,280 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:20:42,280 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:20:46,414 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:20:51,415 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:20:51,689 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:20:51,690 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:20:51,690 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:20:51,691 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:20:52,341 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:20:52,978 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:20:53,342 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:20:56,979 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:20:57,275 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:20:57,280 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:20:57,280 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:21:02,357 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:21:07,358 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:21:12,276 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:21:12,280 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:21:12,281 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:21:12,589 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:21:12,590 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:21:12,591 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:21:12,591 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:21:12,592 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:21:13,349 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:21:15,350 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:21:17,702 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:21:22,703 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:21:22,872 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:21:27,275 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:21:27,280 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:21:27,281 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:21:28,388 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:21:33,389 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:21:36,398 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:21:36,400 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:21:36,400 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:21:36,402 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:21:37,358 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:21:39,178 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:21:39,358 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:21:42,276 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:21:42,281 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:21:42,281 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:21:44,368 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:21:49,369 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:21:52,873 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:21:54,874 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:21:57,276 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:21:57,281 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:21:57,281 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:21:58,996 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:21:58,998 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:21:58,998 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:21:58,998 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:21:59,366 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:22:00,757 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:22:01,367 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:22:05,758 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:22:10,759 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:22:12,276 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:22:12,281 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:22:12,281 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:22:16,386 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:22:21,103 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:22:21,104 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:22:21,104 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:22:21,106 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:22:21,374 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:22:22,107 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:22:22,874 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:22:25,376 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:22:27,276 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:22:27,282 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:22:27,282 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:22:27,403 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:22:32,403 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:22:37,404 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:22:40,901 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:22:40,902 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:22:40,903 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:22:40,903 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:22:41,382 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:22:42,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:22:42,282 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:22:42,282 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:22:43,360 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:22:43,383 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:22:48,361 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:22:52,875 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:22:53,876 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:22:57,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:22:57,281 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:22:57,281 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:22:59,356 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:23:02,483 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:23:02,485 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:23:02,485 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:23:02,485 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:23:03,390 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:23:04,521 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:23:05,391 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:23:09,522 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:23:12,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:23:12,281 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:23:12,282 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:23:15,364 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:23:20,365 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:23:22,531 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:23:22,532 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:23:22,533 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:23:22,533 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:23:22,876 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:23:23,397 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:23:25,398 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:23:25,401 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:23:27,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:23:27,282 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:23:27,282 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:23:31,354 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:23:36,355 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:23:41,355 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:23:42,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:23:42,281 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:23:42,282 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:23:42,665 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:23:42,667 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:23:42,667 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:23:42,667 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:23:43,406 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:23:45,407 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:23:46,461 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:23:51,462 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:23:52,877 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:23:56,878 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:23:57,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:23:57,281 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:23:57,282 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:24:02,324 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:24:03,100 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:24:03,101 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:24:03,101 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:24:03,101 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:24:03,413 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:24:05,414 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:24:07,958 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:24:12,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:24:12,282 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:24:12,282 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:24:13,351 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:24:18,352 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:24:22,878 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:24:23,622 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:24:23,623 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:24:23,624 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:24:23,624 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:24:23,625 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:24:24,421 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:24:25,421 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:24:27,278 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:24:27,282 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:24:27,282 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:24:29,324 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:24:34,325 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:24:39,326 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:24:42,279 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:24:42,282 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:24:42,282 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:24:42,718 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:24:42,719 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:24:42,720 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:24:42,720 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:24:43,429 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:24:44,686 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:24:45,429 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:24:49,687 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:24:52,879 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:24:54,880 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:24:57,279 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:24:57,282 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:24:57,283 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:25:00,405 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:25:05,406 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:25:08,104 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:25:08,105 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:25:08,105 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:25:08,106 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:25:08,438 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:25:09,439 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:25:10,878 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:25:12,279 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:25:12,282 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:25:12,283 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:25:16,389 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:25:21,390 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:25:22,880 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:25:26,881 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:25:27,279 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:25:27,282 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:25:27,283 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:25:29,875 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:25:29,876 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:25:29,876 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:25:29,877 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:25:30,448 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:25:31,449 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:25:32,702 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:25:37,703 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:25:42,279 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:25:42,282 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:25:42,283 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:25:43,337 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:25:48,338 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:25:49,799 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:25:49,800 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:25:49,801 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:25:49,801 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:25:50,456 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:25:51,456 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:25:52,881 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:25:53,882 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:25:57,279 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:25:57,283 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:25:57,283 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:25:59,416 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:26:04,416 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:26:09,417 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:26:10,504 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:26:10,505 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:26:10,506 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:26:10,506 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:26:11,463 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:26:12,279 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:26:12,283 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:26:12,283 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:26:13,464 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:26:15,343 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:26:20,344 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:26:22,882 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:26:25,883 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:26:27,280 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:26:27,283 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:26:27,283 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:26:31,397 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:26:33,658 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:26:33,660 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:26:33,660 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:26:33,661 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:26:34,472 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:26:35,473 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:26:36,486 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:26:41,487 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:26:42,280 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:26:42,283 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:26:42,283 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:26:47,332 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:26:52,333 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:26:52,883 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:26:53,879 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:26:53,881 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:26:53,881 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:26:53,881 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:26:54,481 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:26:55,482 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:26:57,280 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:26:57,283 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:26:57,284 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:26:57,356 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:27:02,357 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:27:07,358 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:27:12,280 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:27:12,283 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:27:12,284 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:27:12,400 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:27:15,674 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:27:15,675 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:27:15,675 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:27:15,676 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:27:16,491 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:27:17,462 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:27:17,491 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:27:22,463 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:27:22,883 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:27:27,280 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:27:27,283 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:27:27,284 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:27:28,322 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:27:33,323 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:27:36,080 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:27:36,081 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:27:36,081 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:27:36,082 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:27:36,500 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:27:37,501 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:27:38,912 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:27:42,280 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:27:42,283 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:27:42,284 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:27:44,340 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:27:49,341 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:27:52,885 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:27:54,886 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:27:55,365 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:27:55,366 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:27:55,367 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:27:55,367 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:27:55,508 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:27:57,280 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:27:57,283 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:27:57,284 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:27:57,509 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:28:00,377 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:28:05,378 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:28:10,379 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:28:12,280 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:28:12,284 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:28:12,284 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:28:16,337 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:28:18,848 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:28:18,849 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:28:18,849 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:28:18,850 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:28:19,517 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:28:21,518 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:28:21,635 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:28:22,886 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:28:26,887 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:28:27,281 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:28:27,284 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:28:27,284 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:28:32,380 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:28:37,381 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:28:40,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:28:40,442 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:28:40,442 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:28:40,443 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:28:40,525 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:28:41,525 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:28:42,281 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:28:42,284 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:28:42,284 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:28:43,354 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:28:48,355 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:28:52,887 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:28:53,888 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:28:57,281 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:28:57,284 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:28:57,284 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:28:59,417 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:29:01,638 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:29:01,639 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:29:01,639 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:29:01,640 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:29:02,534 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:29:03,535 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:29:05,400 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:29:10,401 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:29:12,281 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:29:12,284 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:29:12,284 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:29:15,415 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:29:20,415 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:29:22,490 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:29:22,492 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:29:22,492 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:29:22,492 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:29:22,542 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:29:22,892 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:29:23,542 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:29:26,255 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:29:27,281 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:29:27,284 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:29:27,285 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:29:31,362 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:29:36,363 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:29:41,363 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:29:42,281 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:29:42,284 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:29:42,285 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:29:42,399 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:29:42,401 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:29:42,401 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:29:42,401 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:29:42,549 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:29:43,549 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:29:47,164 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:29:52,165 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:29:52,889 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:29:57,281 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:29:57,285 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:29:57,285 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:29:57,426 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:30:02,427 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:30:03,914 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:30:03,916 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:30:03,916 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:30:03,918 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:30:04,557 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:30:05,557 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:30:07,678 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:30:12,282 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:30:12,284 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:30:12,285 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:30:13,342 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:30:18,343 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:30:22,890 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:30:23,891 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:30:25,641 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:30:25,642 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:30:25,642 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:30:25,643 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:30:26,567 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:30:27,282 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:30:27,285 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:30:27,286 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:30:27,567 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:30:29,384 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:30:34,385 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:30:39,385 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:30:42,282 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:30:42,285 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:30:42,285 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:30:45,377 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:30:47,850 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:30:47,851 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:30:47,852 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:30:47,854 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:30:48,574 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:30:49,575 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:30:50,611 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:30:52,891 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:30:55,892 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:30:57,282 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:30:57,285 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:30:57,285 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:31:01,354 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:31:06,355 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:31:11,215 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:31:11,216 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:31:11,216 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:31:11,217 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:31:11,584 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:31:12,054 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:31:12,282 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:31:12,285 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:31:12,286 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:31:14,585 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:31:17,375 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:31:22,375 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:31:22,892 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:31:27,282 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:31:27,285 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:31:27,286 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:31:27,383 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:31:30,374 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:31:30,376 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:31:30,376 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:31:30,377 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:31:30,591 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:31:32,592 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:31:33,122 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:31:38,123 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:31:42,282 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:31:42,286 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:31:42,286 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:31:43,375 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:31:48,376 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:31:52,216 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:31:52,217 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:31:52,217 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:31:52,218 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:31:52,600 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:31:52,892 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:31:53,993 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:31:54,601 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:31:57,283 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:31:57,286 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:31:57,286 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:31:59,399 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:32:04,400 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:32:09,400 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:32:11,991 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:32:11,992 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:32:11,992 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:32:11,993 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:32:12,480 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:32:12,481 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:32:12,521 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:32:12,608 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:32:14,609 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:32:14,783 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:32:19,784 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:32:22,893 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:32:24,894 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:32:27,425 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:32:27,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:32:27,432 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:32:30,464 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:32:31,421 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:32:31,423 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:32:31,423 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:32:31,423 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:32:31,616 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:32:34,617 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:32:36,188 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:32:41,189 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:32:42,414 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:32:42,429 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:32:42,430 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:32:46,482 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:32:50,372 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:32:50,373 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:32:50,373 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:32:50,374 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:32:50,622 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:32:52,123 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:32:52,623 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:32:52,894 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:32:57,415 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:32:57,417 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:32:57,430 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:32:57,430 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:33:02,514 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:33:07,515 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:33:09,632 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:33:09,633 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:33:09,634 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:33:09,634 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:33:10,630 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:33:12,415 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:33:12,430 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:33:12,430 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:33:12,631 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:33:13,484 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:33:18,485 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:33:22,895 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:33:23,896 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:33:27,415 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:33:27,430 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:33:27,430 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:33:29,115 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:33:29,116 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:33:29,117 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:33:29,117 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:33:29,118 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:33:29,636 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:33:32,638 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:33:34,901 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:33:39,902 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:33:42,415 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:33:42,430 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:33:42,430 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:33:45,486 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:33:50,487 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:33:52,618 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:33:52,619 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:33:52,620 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:33:52,620 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:33:52,646 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:33:52,896 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:33:54,647 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:33:56,390 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:33:57,415 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:33:57,430 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:33:57,431 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:34:01,473 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:34:06,474 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:34:11,475 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:34:12,416 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:34:12,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:34:12,431 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:34:13,168 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:34:13,170 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:34:13,170 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:34:13,171 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:34:13,654 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:34:16,655 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:34:16,931 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:34:21,932 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:34:22,897 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:34:27,416 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:34:27,419 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:34:27,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:34:27,431 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:34:32,521 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:34:34,788 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:34:34,789 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:34:34,789 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:34:34,789 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:34:35,662 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:34:36,663 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:34:37,536 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:34:42,416 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:34:42,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:34:42,431 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:34:43,454 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:34:48,455 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:34:52,897 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:34:53,898 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:34:55,100 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:34:55,102 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:34:55,102 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:34:55,102 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:34:55,670 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:34:57,416 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:34:57,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:34:57,431 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:34:58,671 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:34:59,463 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:35:04,463 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:35:09,464 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:35:12,416 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:35:12,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:35:12,432 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:35:14,591 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:35:16,416 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:35:16,418 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:35:16,418 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:35:16,418 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:35:16,679 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:35:18,679 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:35:20,248 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:35:22,899 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:35:25,900 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:35:27,416 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:35:27,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:35:27,431 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:35:31,557 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:35:36,505 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:35:36,506 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:35:36,506 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:35:36,507 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:35:36,686 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:35:37,259 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:35:38,687 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:35:42,260 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:35:42,416 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:35:42,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:35:42,431 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:35:47,530 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:35:52,531 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:35:52,900 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:35:57,417 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:35:57,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:35:57,432 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:35:57,533 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:36:02,535 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:36:03,213 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:36:03,215 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:36:03,215 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:36:03,215 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:36:03,696 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:36:06,698 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:36:07,965 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:36:12,417 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:36:12,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:36:12,432 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:36:13,476 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:36:18,477 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:36:22,901 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:36:23,901 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:36:24,140 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:36:24,141 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:36:24,141 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:36:24,142 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:36:24,704 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:36:26,705 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:36:27,417 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:36:27,432 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:36:27,433 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:36:29,491 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:36:34,492 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:36:39,493 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:36:42,417 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:36:42,432 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:36:42,432 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:36:44,542 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:36:45,679 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:36:45,680 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:36:45,681 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:36:45,681 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:36:45,712 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:36:48,713 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:36:50,454 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:36:52,901 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:36:55,902 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:36:57,417 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:36:57,432 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:36:57,432 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:37:01,529 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:37:06,530 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:37:07,695 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:37:07,696 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:37:07,696 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:37:07,696 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:37:07,720 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:37:10,722 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:37:12,417 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:37:12,419 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:37:12,432 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:37:12,432 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:37:17,483 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:37:22,484 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:37:22,902 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:37:27,368 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:37:27,369 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:37:27,369 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:37:27,369 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:37:27,417 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:37:27,432 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:37:27,432 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:37:27,552 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:37:27,727 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:37:30,729 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:37:33,347 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:37:38,348 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:37:42,418 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:37:42,432 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:37:42,433 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:37:43,571 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:37:46,766 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:37:46,768 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:37:46,768 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:37:46,768 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:37:47,735 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:37:48,735 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:37:49,559 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:37:52,903 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:37:54,904 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:37:57,418 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:37:57,432 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:37:57,433 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:38:00,555 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:38:05,556 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:38:07,873 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:38:07,874 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:38:07,874 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:38:07,875 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:38:08,743 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:38:10,651 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:38:10,744 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:38:12,418 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:38:12,433 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:38:12,433 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:38:16,499 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:38:21,500 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:38:22,904 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:38:26,905 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:38:27,418 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:38:27,433 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:38:27,433 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:38:29,575 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:38:29,576 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:38:29,576 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:38:29,577 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:38:29,752 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:38:32,341 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:38:32,753 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:38:37,342 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:38:42,343 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:38:42,418 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:38:42,433 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:38:42,433 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:38:47,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:38:51,992 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:38:51,993 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:38:51,993 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:38:51,994 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:38:52,757 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:38:52,759 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:38:52,905 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:38:54,760 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:38:57,418 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:38:57,433 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:38:57,434 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:38:58,498 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:39:03,499 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:39:08,500 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:39:12,419 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:39:12,433 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:39:12,434 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:39:13,514 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:39:14,178 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:39:14,179 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:39:14,179 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:39:14,180 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:39:14,769 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:39:16,770 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:39:18,941 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:39:22,906 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:39:24,907 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:39:27,419 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:39:27,433 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:39:27,434 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:39:30,465 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:39:35,466 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:39:37,046 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:39:37,047 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:39:37,048 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:39:37,048 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:39:37,777 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:39:38,777 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:39:40,811 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:39:42,419 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:39:42,434 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:39:42,434 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:39:46,531 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:39:51,532 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:39:52,907 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:39:56,909 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:39:57,419 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:39:57,433 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:39:57,434 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:39:57,645 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:39:57,647 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:39:57,647 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:39:57,647 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:39:57,784 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:40:00,785 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:40:02,544 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:40:07,545 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:40:12,419 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:40:12,434 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:40:12,434 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:40:12,552 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:40:17,554 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:40:18,290 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:40:18,291 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:40:18,292 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:40:18,292 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:40:18,792 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:40:20,793 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:40:22,909 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:40:22,910 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:40:27,419 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:40:27,434 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:40:27,434 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:40:28,547 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:40:33,547 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:40:38,549 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:40:40,963 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:40:40,964 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:40:40,964 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:40:40,965 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:40:41,803 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:40:42,420 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:40:42,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:40:42,439 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:40:42,803 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:40:43,563 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:40:48,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:40:52,909 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:40:53,910 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:40:57,419 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:40:57,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:40:57,438 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:40:59,538 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:41:01,572 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:41:01,573 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:41:01,574 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:41:01,574 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:41:01,810 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:41:04,811 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:41:05,320 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:41:10,321 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:41:12,419 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:41:12,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:41:12,438 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:41:15,542 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:41:20,544 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:41:22,742 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:41:22,743 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:41:22,743 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:41:22,744 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:41:22,816 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:41:22,910 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:41:24,817 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:41:26,485 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:41:27,420 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:41:27,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:41:27,438 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:41:32,462 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:41:37,463 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:41:42,420 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:41:42,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:41:42,438 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:41:42,555 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:41:47,556 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:41:50,340 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:41:50,341 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:41:50,342 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:41:50,342 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:41:50,826 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:41:52,827 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:41:52,911 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:41:52,912 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:41:57,420 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:41:57,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:41:57,439 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:41:58,548 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:42:03,549 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:42:08,550 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:42:11,306 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:42:11,307 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:42:11,307 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:42:11,308 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:42:11,835 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:42:12,420 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:42:12,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:42:12,439 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:42:12,835 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:42:13,557 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:42:18,558 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:42:22,912 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:42:23,913 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:42:27,420 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:42:27,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:42:27,439 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:42:29,552 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:42:30,890 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:42:30,891 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:42:30,891 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:42:30,891 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:42:31,843 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:42:32,843 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:42:34,665 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:42:39,666 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:42:42,421 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:42:42,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:42:42,439 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:42:45,474 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:42:50,475 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:42:51,304 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:42:51,306 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:42:51,306 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:42:51,306 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:42:51,851 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:42:52,852 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:42:52,913 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:42:55,914 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:42:57,421 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:42:57,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:42:57,439 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:43:01,551 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:43:06,552 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:43:09,328 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:43:09,330 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:43:09,330 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:43:09,331 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:43:09,860 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:43:10,860 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:43:12,283 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:43:12,421 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:43:12,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:43:12,439 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:43:17,523 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:43:22,524 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:43:22,914 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:43:27,421 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:43:27,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:43:27,439 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:43:27,534 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:43:32,535 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:43:33,411 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:43:33,412 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:43:33,412 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:43:33,413 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:43:33,869 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:43:34,869 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:43:38,344 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:43:42,421 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:43:42,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:43:42,439 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:43:43,481 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:43:48,482 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:43:52,916 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:43:53,917 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:43:55,071 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:43:55,072 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:43:55,073 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:43:55,073 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:43:55,878 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:43:56,879 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:43:57,422 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:43:57,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:43:57,439 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:43:59,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:44:04,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:44:09,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:44:12,422 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:44:12,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:44:12,440 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:44:15,487 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:44:17,465 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:44:17,467 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:44:17,467 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:44:17,468 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:44:17,887 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:44:20,888 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:44:21,239 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:44:22,917 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:44:26,918 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:44:27,422 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:44:27,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:44:27,440 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:44:32,482 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:44:37,483 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:44:37,511 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:44:37,512 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:44:37,513 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:44:37,513 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:44:37,895 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:44:40,897 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:44:42,422 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:44:42,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:44:42,440 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:44:43,473 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:44:48,474 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:44:52,917 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:44:53,918 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:44:57,422 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:44:57,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:44:57,440 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:44:58,707 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:44:58,708 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:44:58,709 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:44:58,709 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:44:58,905 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:44:59,480 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:45:00,906 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:45:04,481 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:45:09,482 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:45:12,422 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:45:12,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:45:12,440 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:45:14,544 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:45:18,355 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:45:18,357 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:45:18,357 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:45:18,358 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:45:18,914 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:45:20,174 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:45:20,915 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:45:22,918 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:45:25,919 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:45:27,422 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:45:27,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:45:27,440 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:45:31,524 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:45:36,525 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:45:40,404 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:45:40,405 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:45:40,405 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:45:40,406 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:45:40,924 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:45:42,178 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:45:42,422 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:45:42,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:45:42,440 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:45:42,925 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:45:47,558 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:45:52,559 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:45:52,923 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:45:57,423 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:45:57,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:45:57,441 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:45:58,487 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:46:00,007 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:46:00,008 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:46:00,009 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:46:00,009 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:46:00,932 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:46:02,933 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:46:03,785 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:46:08,786 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:46:12,423 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:46:12,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:46:12,441 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:46:14,462 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:46:19,463 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:46:22,488 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:46:22,490 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:46:22,490 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:46:22,490 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:46:22,921 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:46:22,940 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:46:24,941 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:46:25,239 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:46:27,423 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:46:27,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:46:27,441 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:46:30,558 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:46:35,559 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:46:40,560 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:46:42,423 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:46:42,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:46:42,441 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:46:43,059 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:46:43,060 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:46:43,060 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:46:43,061 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:46:43,948 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:46:44,949 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:46:45,833 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:46:50,834 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:46:52,922 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:46:55,923 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:46:57,430 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:46:57,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:46:57,442 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:47:01,485 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:47:02,625 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:47:02,627 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:47:02,627 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:47:02,629 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:47:02,955 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:47:04,956 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:47:07,369 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:47:12,370 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:47:12,429 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:47:12,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:47:12,441 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:47:17,513 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:47:22,514 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:47:22,923 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:47:24,615 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:47:24,616 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:47:24,616 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:47:24,617 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:47:24,964 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:47:26,965 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:47:27,430 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:47:27,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:47:27,441 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:47:27,561 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:47:32,562 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:47:37,563 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:47:42,429 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:47:42,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:47:42,441 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:47:43,489 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:47:46,318 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:47:46,319 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:47:46,320 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:47:46,322 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:47:46,972 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:47:48,973 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:47:49,101 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:47:52,924 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:47:54,925 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:47:57,430 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:47:57,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:47:57,442 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:48:00,484 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:48:05,486 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:48:07,533 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:48:07,535 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:48:07,535 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:48:07,536 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:48:07,979 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:48:10,981 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:48:11,294 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:48:12,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:48:12,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:48:12,442 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:48:16,476 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:48:21,477 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:48:22,925 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:48:26,926 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:48:27,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:48:27,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:48:27,442 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:48:28,575 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:48:28,576 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:48:28,576 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:48:28,577 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:48:28,987 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:48:30,989 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:48:32,500 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:48:37,501 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:48:42,430 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:48:42,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:48:42,442 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:48:42,555 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:48:47,034 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:48:47,035 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:48:47,035 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:48:47,036 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:48:47,775 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:48:47,996 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:48:48,997 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:48:52,775 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:48:52,926 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:48:57,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:48:57,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:48:57,442 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:48:58,517 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:49:03,517 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:49:05,887 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:49:05,888 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:49:05,888 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:49:05,889 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:49:06,004 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:49:08,726 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:49:09,005 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:49:12,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:49:12,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:49:12,442 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:49:14,547 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:49:19,548 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:49:22,927 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:49:24,928 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:49:27,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:49:27,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:49:27,442 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:49:29,072 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:49:29,073 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:49:29,073 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:49:29,074 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:49:30,012 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:49:30,886 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:49:31,013 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:49:35,887 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:49:40,888 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:49:42,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:49:42,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:49:42,443 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:49:46,490 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:49:51,491 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:49:52,928 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:49:53,001 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:49:53,002 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:49:53,002 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:49:53,003 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:49:53,022 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:49:55,023 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:49:56,769 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:49:57,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:49:57,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:49:57,443 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:50:02,583 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:50:07,584 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:50:12,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:50:12,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:50:12,443 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:50:13,493 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:50:15,939 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:50:15,940 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:50:15,940 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:50:15,941 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:50:16,031 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:50:18,725 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:50:19,032 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:50:22,928 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:50:23,929 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:50:27,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:50:27,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:50:27,443 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:50:29,470 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:50:34,200 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:50:34,202 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:50:34,202 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:50:34,203 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:50:34,979 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:50:35,039 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:50:37,040 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:50:39,980 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:50:42,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:50:42,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:50:42,443 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:50:45,545 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:50:50,546 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:50:52,929 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:50:55,930 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:50:57,432 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:50:57,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:50:57,443 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:50:59,034 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:50:59,035 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:50:59,036 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:50:59,036 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:50:59,048 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:51:01,049 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:51:01,780 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:51:06,781 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:51:11,782 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:51:12,432 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:51:12,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:51:12,443 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:51:17,541 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:51:19,012 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:51:19,013 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:51:19,013 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:51:19,014 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:51:19,056 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:51:21,057 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:51:22,774 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:51:22,930 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:51:27,432 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:51:27,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:51:27,443 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:51:28,525 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:51:33,526 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:51:38,527 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:51:40,674 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:51:40,676 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:51:40,676 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:51:40,676 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:51:41,064 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:51:42,432 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:51:42,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:51:42,444 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:51:43,065 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:51:44,503 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:51:49,504 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:51:52,931 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:51:54,932 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:51:57,433 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:51:57,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:51:57,444 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:52:00,486 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:52:03,140 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:52:03,141 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:52:03,141 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:52:03,142 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:52:04,073 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:52:05,073 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:52:05,899 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:52:10,901 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:52:12,432 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:52:12,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:52:12,444 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:52:16,572 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:52:21,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:52:22,932 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:52:25,101 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:52:25,102 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:52:25,102 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:52:25,103 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:52:26,081 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:52:26,874 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:52:27,082 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:52:27,432 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:52:27,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:52:27,444 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:52:32,498 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:52:37,500 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:52:42,434 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:52:42,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:52:42,444 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:52:43,495 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:52:47,112 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:52:47,113 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:52:47,114 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:52:47,114 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:52:48,090 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:52:48,902 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:52:49,090 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:52:52,934 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:52:53,934 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:52:57,433 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:52:57,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:52:57,445 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:52:59,562 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:53:04,563 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:53:07,833 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:53:07,835 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:53:07,835 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:53:07,835 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:53:08,098 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:53:09,098 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:53:09,590 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:53:12,433 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:53:12,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:53:12,445 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:53:15,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:53:20,569 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:53:22,935 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:53:25,936 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:53:27,433 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:53:27,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:53:27,445 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:53:31,079 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:53:31,080 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:53:31,080 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:53:31,080 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:53:31,081 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:53:31,107 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:53:33,107 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:53:36,088 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:53:41,089 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:53:42,433 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:53:42,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:53:42,445 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:53:46,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:53:51,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:53:52,936 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:53:54,237 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:53:54,238 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:53:54,239 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:53:54,239 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:53:55,116 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:53:57,104 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:53:57,117 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:53:57,437 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:53:57,454 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:53:57,454 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:54:02,483 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:54:07,484 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:54:12,437 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:54:12,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:54:12,445 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:54:12,508 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:54:14,031 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:54:14,032 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:54:14,033 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:54:14,033 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:54:14,124 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:54:17,793 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:54:18,126 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:54:22,794 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:54:22,936 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:54:27,437 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:54:27,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:54:27,445 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:54:28,495 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:54:31,679 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:54:31,680 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:54:31,681 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:54:31,681 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:54:32,132 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:54:34,133 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:54:34,431 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:54:39,432 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:54:42,437 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:54:42,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:54:42,445 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:54:44,558 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:54:49,558 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:54:52,693 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:54:52,695 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:54:52,695 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:54:52,695 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:54:52,937 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:54:53,140 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:54:55,470 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:54:56,141 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:54:57,437 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:54:57,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:54:57,446 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:55:00,572 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:55:05,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:55:10,574 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:55:12,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:55:12,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:55:12,446 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:55:12,551 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:55:12,553 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:55:12,553 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:55:12,553 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:55:13,148 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:55:16,149 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:55:16,326 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:55:21,327 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:55:22,938 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:55:26,940 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:55:27,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:55:27,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:55:27,446 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:55:32,513 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:55:34,450 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:55:34,451 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:55:34,452 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:55:34,452 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:55:35,156 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:55:38,158 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:55:38,199 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:55:42,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:55:42,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:55:42,446 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:55:43,577 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:55:48,578 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:55:52,939 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:55:53,940 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:55:57,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:55:57,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:55:57,446 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:55:58,499 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:55:58,500 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:55:58,500 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:55:58,501 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:55:59,166 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:55:59,253 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:56:02,167 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:56:04,254 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:56:09,254 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:56:12,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:56:12,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:56:12,446 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:56:14,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:56:19,161 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:56:19,163 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:56:19,163 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:56:19,164 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:56:19,174 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:56:19,948 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:56:22,176 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:56:22,940 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:56:25,941 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:56:27,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:56:27,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:56:27,446 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:56:31,575 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:56:36,576 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:56:39,347 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:56:39,348 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:56:39,348 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:56:39,349 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:56:40,183 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:56:42,107 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:56:42,184 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:56:42,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:56:42,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:56:42,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:56:47,555 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:56:52,555 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:56:52,941 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:56:57,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:56:57,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:56:57,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:56:58,535 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:57:01,046 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:57:01,047 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:57:01,047 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:57:01,048 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:57:01,191 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:57:03,811 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:57:04,192 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:57:08,811 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:57:12,438 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:57:12,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:57:12,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:57:14,497 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:57:19,498 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:57:21,033 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:57:21,035 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:57:21,035 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:57:21,036 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:57:21,198 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:57:22,942 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:57:24,200 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:57:24,943 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:57:27,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:57:27,446 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:57:27,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:57:30,572 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:57:35,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:57:39,919 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:57:39,920 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:57:39,920 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:57:39,921 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:57:40,207 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:57:40,677 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:57:42,207 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:57:42,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:57:42,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:57:42,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:57:46,524 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:57:51,525 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:57:52,943 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:57:56,944 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:57:57,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:57:57,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:57:57,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:57:59,591 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:57:59,593 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:57:59,593 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:57:59,593 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:58:00,215 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:58:02,122 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:58:04,217 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:58:07,123 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:58:12,124 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:58:12,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:58:12,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:58:12,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:58:17,526 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:58:22,097 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:58:22,098 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:58:22,098 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:58:22,099 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:58:22,224 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:58:22,855 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:58:22,944 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:58:24,225 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:58:27,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:58:27,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:58:27,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:58:28,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:58:33,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:58:38,569 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:58:39,984 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:58:39,986 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:58:39,986 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:58:39,986 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:58:40,231 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:58:42,232 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:58:42,439 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:58:42,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:58:42,448 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:58:44,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:58:49,568 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:58:52,945 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:58:54,946 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:58:57,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:58:57,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:58:57,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:59:00,513 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:59:03,485 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:59:03,486 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:59:03,486 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:59:03,487 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:59:04,242 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:59:06,242 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:59:06,251 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:59:11,251 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:59:12,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:59:12,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:59:12,448 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:59:16,490 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:59:21,490 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:59:22,946 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:59:24,275 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:59:24,277 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:59:24,277 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:59:24,278 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:59:25,249 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:59:26,249 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:59:27,044 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:59:27,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:59:27,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:59:27,448 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:59:32,514 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:59:37,515 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:59:42,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:59:42,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:59:42,448 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:59:42,576 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:59:46,489 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 16:59:46,490 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 16:59:46,490 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 16:59:46,491 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 16:59:47,257 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 16:59:48,239 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:59:50,258 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 16:59:52,947 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 16:59:53,948 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 16:59:57,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 16:59:57,447 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 16:59:57,448 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 16:59:59,488 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:00:04,489 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:00:07,525 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:00:07,526 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:00:07,526 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:00:07,527 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:00:08,265 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:00:10,266 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:00:10,313 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:00:12,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:00:12,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:00:12,448 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:00:15,562 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:00:20,563 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:00:22,948 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:00:25,949 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:00:27,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:00:27,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:00:27,448 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:00:29,907 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:00:29,908 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:00:29,909 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:00:29,909 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:00:30,273 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:00:31,655 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:00:32,274 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:00:36,656 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:00:41,657 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:00:42,440 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:00:42,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:00:42,448 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:00:47,549 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:00:50,367 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:00:50,369 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:00:50,369 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:00:50,369 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:00:51,280 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:00:52,281 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:00:52,949 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:00:52,950 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:00:57,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:00:57,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:00:57,448 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:00:58,550 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:01:03,551 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:01:08,552 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:01:08,604 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:01:08,606 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:01:08,606 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:01:08,606 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:01:09,288 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:01:12,289 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:01:12,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:01:12,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:01:12,449 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:01:13,596 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:01:18,597 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:01:22,950 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:01:23,951 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:01:27,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:01:27,449 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:01:27,449 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:01:29,530 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:01:30,653 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:01:30,654 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:01:30,655 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:01:30,655 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:01:31,296 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:01:34,297 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:01:35,428 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:01:40,429 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:01:42,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:01:42,449 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:01:42,449 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:01:45,549 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:01:50,550 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:01:51,890 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:01:51,891 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:01:51,891 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:01:51,892 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:01:52,304 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:01:52,951 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:01:54,305 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:01:55,952 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:01:57,441 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:01:57,449 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:01:57,449 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:02:01,582 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:02:06,583 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:02:11,584 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:02:12,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:02:12,449 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:02:12,449 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:02:12,656 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:02:12,657 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:02:12,657 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:02:12,658 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:02:13,312 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:02:16,313 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:02:16,595 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:02:21,596 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:02:22,952 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:02:26,953 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:02:27,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:02:27,449 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:02:27,450 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:02:32,560 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:02:33,061 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:02:33,062 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:02:33,062 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:02:33,063 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:02:33,320 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:02:36,321 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:02:37,849 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:02:42,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:02:42,449 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:02:42,450 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:02:43,480 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:02:48,481 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:02:51,088 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:02:51,090 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:02:51,090 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:02:51,090 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:02:51,327 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:02:52,953 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:02:53,953 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:02:54,329 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:02:57,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:02:57,450 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:02:57,450 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:02:59,474 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:03:04,475 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:03:09,476 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:03:09,523 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:03:09,524 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:03:09,525 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:03:09,525 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:03:10,335 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:03:12,336 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:03:12,442 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:03:12,449 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:03:12,450 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:03:14,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:03:19,574 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:03:22,953 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:03:24,954 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:03:27,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:03:27,449 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:03:27,450 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:03:30,531 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:03:32,215 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:03:32,217 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:03:32,217 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:03:32,217 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:03:32,344 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:03:34,345 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:03:36,062 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:03:41,063 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:03:42,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:03:42,450 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:03:42,450 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:03:46,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:03:51,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:03:52,954 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:03:54,276 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:03:54,277 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:03:54,277 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:03:54,278 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:03:54,353 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:03:56,354 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:03:57,034 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:03:57,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:03:57,450 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:03:57,450 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:04:02,476 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:04:07,477 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:04:12,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:04:12,450 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:04:12,450 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:04:12,482 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:04:17,483 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:04:18,008 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:04:18,009 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:04:18,009 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:04:18,010 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:04:18,363 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:04:20,365 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:04:22,768 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:04:22,955 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:04:27,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:04:27,450 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:04:27,450 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:04:28,515 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:04:33,516 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:04:38,281 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:04:38,283 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:04:38,283 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:04:38,283 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:04:38,371 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:04:39,141 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:04:40,372 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:04:42,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:04:42,450 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:04:42,450 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:04:44,521 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:04:49,522 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:04:52,957 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:04:54,958 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:04:57,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:04:57,450 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:04:57,451 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:05:00,012 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:05:00,014 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:05:00,014 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:05:00,014 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:05:00,015 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:05:00,382 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:05:02,383 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:05:05,792 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:05:10,793 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:05:12,443 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:05:12,450 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:05:12,451 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:05:16,567 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:05:19,738 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:05:19,739 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:05:19,739 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:05:19,740 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:05:20,389 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:05:22,390 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:05:22,493 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:05:22,957 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:05:27,444 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:05:27,450 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:05:27,451 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:05:27,556 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:05:32,558 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:05:37,558 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:05:41,734 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:05:41,735 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:05:41,736 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:05:41,736 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:05:42,398 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:05:42,464 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:05:42,466 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:05:42,467 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:05:42,589 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:05:44,398 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:05:47,589 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:05:52,591 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:05:52,958 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:05:57,462 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:05:57,462 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:05:57,463 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:05:58,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:06:03,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:06:05,480 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:06:05,482 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:06:05,482 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:06:05,482 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:06:06,408 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:06:08,409 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:06:09,226 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:06:12,461 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:06:12,461 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:06:12,464 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:06:14,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:06:19,571 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:06:22,959 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:06:24,960 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:06:25,445 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:06:25,446 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:06:25,447 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:06:25,447 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:06:26,415 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:06:27,461 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:06:27,461 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:06:27,462 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:06:28,417 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:06:30,515 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:06:35,516 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:06:40,517 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:06:42,461 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:06:42,461 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:06:42,502 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:06:45,525 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:06:49,860 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:06:49,861 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:06:49,862 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:06:49,862 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:06:50,425 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:06:50,657 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:06:52,426 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:06:52,960 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:06:55,961 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:06:57,461 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:06:57,461 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:06:57,462 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:07:01,589 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:07:06,590 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:07:11,591 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:07:12,461 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:07:12,461 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:07:12,502 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:07:13,351 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:07:13,352 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:07:13,353 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:07:13,353 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:07:13,434 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:07:16,435 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:07:17,110 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:07:22,111 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:07:22,961 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:07:27,461 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:07:27,462 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:07:27,462 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:07:27,556 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:07:32,557 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:07:36,686 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:07:36,687 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:07:36,688 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:07:36,688 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:07:37,444 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:07:38,444 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:07:38,481 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:07:42,462 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:07:42,463 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:07:42,467 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:07:43,493 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:07:48,494 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:07:52,962 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:07:53,962 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:07:56,677 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:07:56,678 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:07:56,679 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:07:56,679 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:07:57,452 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:07:57,462 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:07:57,462 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:07:57,464 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:07:58,452 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:07:59,577 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:08:04,578 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:08:09,579 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:08:12,461 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:08:12,462 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:08:12,503 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:08:15,545 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:08:17,547 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:08:17,548 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:08:17,548 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:08:17,549 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:08:18,461 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:08:20,462 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:08:21,299 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:08:22,963 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:08:26,964 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:08:27,462 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:08:27,462 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:08:27,463 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:08:32,543 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:08:37,544 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:08:41,530 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:08:41,531 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:08:41,532 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:08:41,532 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:08:42,462 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:08:42,462 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:08:42,464 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:08:42,472 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:08:43,538 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:08:44,473 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:08:48,539 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:08:52,963 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:08:53,964 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:08:57,462 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:08:57,462 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:08:57,503 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:08:59,589 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:09:02,328 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:09:02,329 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:09:02,329 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:09:02,330 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:09:02,480 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:09:04,481 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:09:05,173 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:09:10,174 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:09:12,462 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:09:12,463 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:09:12,463 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:09:15,529 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:09:20,530 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:09:22,965 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:09:25,966 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:09:27,462 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:09:27,463 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:09:27,503 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:09:28,373 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:09:28,375 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:09:28,375 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:09:28,375 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:09:28,489 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:09:30,490 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:09:31,140 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:09:36,141 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:09:41,142 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:09:42,463 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:09:42,463 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:09:42,464 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:09:46,587 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:09:47,891 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:09:47,892 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:09:47,892 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:09:47,893 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:09:48,498 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:09:50,499 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:09:51,656 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:09:52,965 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:09:56,967 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:09:57,463 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:09:57,463 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:09:57,464 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:10:02,583 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:10:07,583 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:10:09,993 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:10:09,995 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:10:09,995 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:10:09,995 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:10:10,506 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:10:12,463 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:10:12,463 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:10:12,466 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:10:12,506 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:10:13,502 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:10:18,503 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:10:22,966 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:10:23,967 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:10:27,463 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:10:27,463 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:10:27,504 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:10:29,551 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:10:32,056 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:10:32,057 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:10:32,058 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:10:32,058 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:10:32,514 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:10:34,515 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:10:34,911 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:10:39,912 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:10:42,463 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:10:42,464 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:10:42,466 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:10:45,511 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:10:50,511 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:10:52,847 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:10:52,849 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:10:52,849 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:10:52,850 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:10:52,967 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:10:53,523 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:10:54,523 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:10:55,626 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:10:57,463 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:10:57,464 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:10:57,464 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:11:01,522 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:11:06,523 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:11:11,524 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:11:12,463 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:11:12,464 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:11:12,485 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:11:13,484 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:11:13,485 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:11:13,485 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:11:13,486 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:11:13,530 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:11:16,531 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:11:17,261 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:11:22,262 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:11:22,968 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:11:27,464 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:11:27,464 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:11:27,465 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:11:27,599 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:11:32,599 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:11:36,951 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:11:36,952 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:11:36,953 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:11:36,953 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:11:37,540 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:11:37,851 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:11:38,541 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:11:42,464 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:11:42,464 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:11:42,465 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:11:43,533 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:11:48,534 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:11:52,970 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:11:53,971 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:11:55,870 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:11:55,871 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:11:55,871 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:11:55,872 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:11:56,548 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:11:57,465 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:11:57,465 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:11:57,466 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:11:58,549 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:11:59,552 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:12:04,553 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:12:09,554 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:12:12,464 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:12:12,465 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:12:12,505 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:12:14,586 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:12:18,527 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:12:18,528 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:12:18,528 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:12:18,529 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:12:18,557 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:12:20,292 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:12:20,557 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:12:22,972 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:12:25,973 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:12:27,464 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:12:27,465 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:12:27,465 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:12:31,528 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:12:36,529 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:12:39,793 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:12:39,794 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:12:39,795 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:12:39,795 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:12:40,565 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:12:41,552 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:12:42,464 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:12:42,465 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:12:42,466 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:12:42,566 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:12:47,497 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:12:52,497 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:12:52,972 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:12:57,464 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:12:57,465 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:12:57,505 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:12:57,597 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:13:02,463 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:13:02,464 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:13:02,465 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:13:02,465 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:13:02,574 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:13:03,231 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:13:04,574 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:13:08,232 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:13:12,465 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:13:12,465 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:13:12,466 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:13:13,514 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:13:18,515 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:13:22,482 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:13:22,483 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:13:22,484 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:13:22,484 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:13:22,581 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:13:22,973 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:13:24,224 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:13:24,582 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:13:27,471 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:13:27,472 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:13:27,473 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:13:29,514 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:13:34,515 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:13:39,516 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:13:42,471 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:13:42,472 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:13:42,472 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:13:44,308 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:13:44,309 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:13:44,309 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:13:44,310 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:13:44,590 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:13:45,081 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:13:46,591 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:13:50,082 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:13:52,974 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:13:55,975 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:13:57,471 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:13:57,472 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:13:57,473 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:14:01,550 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:14:03,145 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:14:03,146 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:14:03,146 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:14:03,147 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:14:03,597 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:14:04,598 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:14:06,895 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:14:11,896 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:14:12,470 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:14:12,471 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:14:12,471 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:14:17,534 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:14:22,534 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:14:22,975 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:14:25,834 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:14:25,836 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:14:25,836 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:14:25,836 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:14:26,606 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:14:27,470 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:14:27,471 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:14:27,472 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:14:27,552 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:14:28,607 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:14:32,553 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:14:37,554 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:14:42,471 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:14:42,471 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:14:42,472 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:14:42,558 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:14:47,559 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:14:50,978 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:14:50,979 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:14:50,980 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:14:50,980 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:14:51,615 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:14:52,615 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:14:52,901 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:14:52,976 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:14:57,471 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:14:57,472 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:14:57,473 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:14:58,590 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:15:03,591 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:15:08,592 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:15:12,471 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:15:12,472 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:15:12,473 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:15:13,204 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:15:13,205 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:15:13,205 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:15:13,206 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:15:13,624 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:15:14,206 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:15:14,625 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:15:19,293 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:15:22,977 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:15:24,978 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:15:27,471 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:15:27,472 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:15:27,472 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:15:30,518 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:15:34,141 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:15:34,142 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:15:34,143 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:15:34,143 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:15:34,633 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:15:36,026 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:15:36,634 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:15:41,027 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:15:42,472 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:15:42,473 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:15:42,474 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:15:46,562 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:15:51,563 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:15:52,978 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:15:54,322 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:15:54,323 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:15:54,323 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:15:54,324 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:15:54,641 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:15:56,642 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:15:57,080 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:15:57,472 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:15:57,474 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:15:57,475 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:16:02,502 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:16:07,503 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:16:12,472 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:16:12,473 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:16:12,474 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:16:12,552 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:16:14,753 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:16:14,754 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:16:14,754 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:16:14,755 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:16:15,649 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:16:16,650 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:16:18,504 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:16:22,979 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:16:23,980 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:16:27,472 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:16:27,473 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:16:27,474 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:16:29,498 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:16:34,499 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:16:37,049 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:16:37,050 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:16:37,050 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:16:37,051 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:16:37,658 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:16:38,659 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:16:39,832 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:16:42,472 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:16:42,474 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:16:42,474 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:16:45,536 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:16:50,537 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:16:52,980 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:16:55,981 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:16:57,071 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:16:57,073 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:16:57,073 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:16:57,073 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:16:57,544 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:16:57,545 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:16:57,582 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:16:57,666 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:16:58,668 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:17:01,869 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:17:06,870 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:17:11,870 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:17:12,499 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:17:12,500 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:17:12,500 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:17:17,616 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:17:18,599 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:17:18,600 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:17:18,600 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:17:18,601 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:17:18,676 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:17:21,677 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:17:22,981 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:17:22,983 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:17:27,499 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:17:27,500 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:17:27,501 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:17:28,621 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:17:33,622 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:17:38,252 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:17:38,253 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:17:38,254 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:17:38,254 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:17:38,683 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:17:39,008 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:17:41,684 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:17:42,499 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:17:42,500 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:17:42,500 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:17:44,608 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:17:49,609 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:17:52,982 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:17:54,983 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:17:57,500 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:17:57,500 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:17:57,501 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:17:58,981 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:17:58,982 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:17:58,983 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:17:58,983 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:17:59,691 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:18:00,742 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:18:01,692 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:18:05,743 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:18:10,744 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:18:12,500 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:18:12,500 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:18:12,501 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:18:16,538 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:18:21,130 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:18:21,131 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:18:21,131 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:18:21,132 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:18:21,700 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:18:21,897 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:18:22,983 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:18:23,701 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:18:26,984 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:18:27,500 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:18:27,500 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:18:27,501 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:18:32,530 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:18:37,531 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:18:39,832 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:18:39,834 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:18:39,834 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:18:39,834 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:18:40,707 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:18:42,500 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:18:42,501 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:18:42,501 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:18:42,617 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:18:43,709 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:18:47,618 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:18:52,618 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:18:52,987 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:18:57,500 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:18:57,500 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:18:57,501 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:18:58,541 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:19:01,498 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:19:01,500 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:19:01,500 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:19:01,500 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:19:01,716 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:19:03,716 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:19:04,262 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:19:09,263 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:19:12,501 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:19:12,502 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:19:12,502 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:19:14,622 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:19:19,624 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:19:22,984 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:19:24,170 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:19:24,172 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:19:24,172 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:19:24,174 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:19:24,725 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:19:24,924 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:19:27,501 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:19:27,501 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:19:27,502 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:19:27,726 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:19:30,577 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:19:35,578 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:19:40,579 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:19:42,501 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:19:42,501 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:19:42,502 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:19:44,955 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:19:44,957 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:19:44,957 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:19:44,957 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:19:45,733 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:19:45,751 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:19:47,733 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:19:50,753 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:19:52,985 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:19:55,986 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:19:57,501 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:19:57,501 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:19:57,502 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:20:01,620 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:20:03,766 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:20:03,768 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:20:03,768 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:20:03,770 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:20:04,739 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:20:05,740 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:20:07,533 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:20:12,501 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:20:12,501 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:20:12,502 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:20:12,548 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:20:17,549 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:20:22,550 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:20:22,986 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:20:27,082 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:20:27,083 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:20:27,084 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:20:27,084 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:20:27,560 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:20:27,566 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:20:27,578 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:20:27,579 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:20:27,748 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:20:29,749 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:20:32,866 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:20:37,867 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:20:42,507 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:20:42,507 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:20:42,508 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:20:43,556 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:20:48,557 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:20:48,882 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:20:48,883 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:20:48,883 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:20:48,885 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:20:49,757 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:20:51,758 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:20:52,987 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:20:53,988 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:20:57,507 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:20:57,507 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:20:57,508 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:20:59,589 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:21:04,590 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:21:09,591 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:21:11,266 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:21:11,268 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:21:11,268 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:21:11,268 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:21:11,766 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:21:12,507 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:21:12,507 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:21:12,508 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:21:13,767 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:21:15,558 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:21:20,559 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:21:22,988 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:21:25,989 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:21:27,508 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:21:27,508 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:21:27,509 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:21:31,560 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:21:34,149 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:21:34,150 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:21:34,150 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:21:34,151 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:21:34,774 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:21:36,955 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:21:37,776 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:21:41,956 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:21:42,508 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:21:42,508 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:21:42,509 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:21:47,607 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:21:52,608 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:21:52,989 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:21:56,570 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:21:56,572 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:21:56,572 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:21:56,573 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:21:56,783 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:21:57,509 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:21:57,510 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:21:57,510 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:21:58,556 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:21:59,785 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:22:03,557 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:22:08,559 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:22:12,508 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:22:12,508 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:22:12,509 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:22:13,578 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:22:18,578 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:22:21,672 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:22:21,674 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:22:21,674 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:22:21,674 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:22:21,793 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:22:22,990 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:22:23,794 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:22:23,991 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:22:27,508 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:22:27,509 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:22:27,510 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:22:29,602 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:22:34,603 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:22:39,604 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:22:42,508 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:22:42,509 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:22:42,509 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:22:42,976 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:22:42,977 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:22:42,978 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:22:42,978 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:22:43,801 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:22:44,904 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:22:45,802 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:22:49,905 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:22:52,991 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:22:54,992 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:22:57,508 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:22:57,509 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:22:57,509 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:23:00,540 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:23:00,877 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:23:00,878 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:23:00,878 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:23:00,879 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:23:01,808 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:23:03,808 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:23:05,629 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:23:10,630 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:23:12,510 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:23:12,510 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:23:12,511 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:23:16,608 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:23:19,448 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:23:19,450 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:23:19,450 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:23:19,450 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:23:19,814 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:23:21,815 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:23:22,249 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:23:22,992 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:23:27,510 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:23:27,511 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:23:27,511 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:23:27,631 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:23:32,632 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:23:37,633 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:23:41,327 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:23:41,329 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:23:41,329 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:23:41,330 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:23:41,823 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:23:42,509 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:23:42,509 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:23:42,510 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:23:43,560 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:23:43,824 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:23:48,561 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:23:52,993 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:23:53,994 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:23:57,509 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:23:57,510 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:23:57,512 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:23:59,557 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:24:02,477 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:24:02,478 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:24:02,479 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:24:02,479 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:24:02,832 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:24:05,247 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:24:05,833 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:24:10,248 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:24:12,509 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:24:12,510 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:24:12,510 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:24:15,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:24:20,574 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:24:20,919 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:24:20,920 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:24:20,920 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:24:20,921 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:24:21,839 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:24:22,994 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:24:23,840 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:24:25,995 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:24:27,510 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:24:27,510 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:24:27,513 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:24:31,573 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:24:36,574 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:24:40,783 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:24:40,785 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:24:40,785 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:24:40,785 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:24:40,846 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:24:41,786 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:24:42,525 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:24:42,526 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:24:42,526 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:24:43,847 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:24:47,562 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:24:52,563 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:24:52,995 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:24:57,510 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:24:57,510 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:24:57,550 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:24:57,621 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:24:58,599 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:24:58,600 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:24:58,600 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:24:58,601 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:24:58,853 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:25:01,854 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:25:03,359 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:25:08,361 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:25:12,510 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:25:12,510 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:25:12,511 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:25:13,583 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:25:18,584 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:25:21,636 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:25:21,638 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:25:21,638 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:25:21,639 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:25:21,861 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:25:22,995 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:25:23,862 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:25:23,996 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:25:27,510 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:25:27,511 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:25:27,511 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:25:29,650 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:25:34,651 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:25:39,652 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:25:42,510 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:25:42,511 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:25:42,512 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:25:44,083 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:25:44,084 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:25:44,084 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:25:44,085 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:25:44,869 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:25:44,919 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:25:45,870 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:25:49,920 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:25:52,996 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:25:54,997 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:25:57,510 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:25:57,511 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:25:57,512 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:26:00,615 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:26:05,616 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:26:06,123 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:26:06,124 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:26:06,124 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:26:06,125 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:26:06,878 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:26:07,878 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:26:10,950 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:26:12,511 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:26:12,511 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:26:12,512 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:26:16,626 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:26:21,628 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:26:22,997 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:26:26,998 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:26:27,511 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:26:27,511 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:26:27,512 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:26:27,640 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:26:27,641 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:26:27,641 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:26:27,642 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:26:27,885 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:26:29,886 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:26:32,409 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:26:37,410 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:26:42,411 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:26:42,511 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:26:42,511 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:26:42,515 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:26:47,599 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:26:49,432 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:26:49,433 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:26:49,433 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:26:49,434 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:26:49,893 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:26:51,894 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:26:53,100 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:26:53,101 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:26:57,511 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:26:57,511 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:26:57,512 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:26:58,625 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:27:03,626 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:27:08,627 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:27:10,821 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:27:10,822 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:27:10,822 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:27:10,823 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:27:10,902 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:27:12,511 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:27:12,511 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:27:12,512 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:27:13,904 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:27:14,564 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:27:19,565 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:27:22,999 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:27:25,000 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:27:27,511 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:27:27,512 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:27:27,515 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:27:30,589 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:27:30,835 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:27:30,836 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:27:30,836 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:27:30,838 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:27:30,911 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:27:33,912 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log -2024-04-12 17:27:35,615 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:27:40,616 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:27:42,511 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status -2024-04-12 17:27:42,512 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-12 17:27:42,512 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: stop_status -2024-04-12 17:27:46,598 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:27:51,599 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report -2024-04-12 17:27:52,350 DEBUG HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history -2024-04-12 17:27:52,351 DEBUG SenderThread:151 [sender.py:send():379] send: history -2024-04-12 17:27:52,351 DEBUG SenderThread:151 [sender.py:send_request():406] send_request: summary_record -2024-04-12 17:27:52,351 INFO SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-12 17:27:52,919 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json -2024-04-12 17:27:53,000 DEBUG SenderThread:151 [sender.py:send():379] send: stats -2024-04-12 17:27:53,919 INFO Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log +2024-04-13 02:11:10,712 DEBUG HandlerThread:145 [system_info.py:_save_conda():222] Saving conda packages done +2024-04-13 02:11:10,713 INFO HandlerThread:145 [system_monitor.py:probe():229] Finished publishing system info +2024-04-13 02:11:10,720 DEBUG HandlerThread:145 [handler.py:handle_request():146] handle_request: status_report +2024-04-13 02:11:10,720 DEBUG HandlerThread:145 [handler.py:handle_request():146] handle_request: keepalive +2024-04-13 02:11:10,720 DEBUG HandlerThread:145 [handler.py:handle_request():146] handle_request: status_report +2024-04-13 02:11:10,720 DEBUG HandlerThread:145 [handler.py:handle_request():146] handle_request: keepalive +2024-04-13 02:11:10,720 DEBUG HandlerThread:145 [handler.py:handle_request():146] handle_request: status_report +2024-04-13 02:11:10,720 DEBUG HandlerThread:145 [handler.py:handle_request():146] handle_request: keepalive +2024-04-13 02:11:10,721 DEBUG SenderThread:145 [sender.py:send():379] send: files +2024-04-13 02:11:10,721 INFO SenderThread:145 [sender.py:_save_file():1390] saving file wandb-metadata.json with policy now +2024-04-13 02:11:10,915 INFO wandb-upload_0:145 [upload_job.py:push():131] Uploaded file /tmp/tmp05iica_pwandb/lq1mybdr-wandb-metadata.json +2024-04-13 02:11:11,572 INFO Thread-12 :145 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240413_021054-hjdgsc19/files/wandb-metadata.json +2024-04-13 02:11:11,697 DEBUG HandlerThread:145 [handler.py:handle_request():146] handle_request: python_packages +2024-04-13 02:11:11,698 DEBUG SenderThread:145 [sender.py:send_request():406] send_request: python_packages +2024-04-13 02:11:11,702 DEBUG SenderThread:145 [sender.py:send():379] send: telemetry +2024-04-13 02:11:11,712 DEBUG HandlerThread:145 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-13 02:11:11,713 DEBUG SenderThread:145 [sender.py:send():379] send: config +2024-04-13 02:11:11,713 DEBUG HandlerThread:145 [handler.py:handle_request():146] handle_request: stop_status +2024-04-13 02:11:11,715 DEBUG SenderThread:145 [sender.py:send():379] send: metric +2024-04-13 02:11:11,716 DEBUG SenderThread:145 [sender.py:send():379] send: telemetry +2024-04-13 02:11:11,716 DEBUG SenderThread:145 [sender.py:send_request():406] send_request: stop_status +2024-04-13 02:11:11,823 DEBUG SenderThread:145 [sender.py:send():379] send: metric +2024-04-13 02:11:11,823 WARNING SenderThread:145 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) +2024-04-13 02:11:11,823 DEBUG SenderThread:145 [sender.py:send():379] send: telemetry +2024-04-13 02:11:12,572 INFO Thread-12 :145 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240413_021054-hjdgsc19/files/requirements.txt +2024-04-13 02:11:13,573 INFO Thread-12 :145 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240413_021054-hjdgsc19/files/output.log +2024-04-13 02:11:15,574 INFO Thread-12 :145 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240413_021054-hjdgsc19/files/output.log +2024-04-13 02:11:15,726 DEBUG HandlerThread:145 [handler.py:handle_request():146] handle_request: status_report +2024-04-13 02:11:20,727 DEBUG HandlerThread:145 [handler.py:handle_request():146] handle_request: status_report +2024-04-13 02:11:25,733 DEBUG HandlerThread:145 [handler.py:handle_request():146] handle_request: status_report +2024-04-13 02:11:26,578 INFO Thread-12 :145 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240413_021054-hjdgsc19/files/config.yaml +2024-04-13 02:11:26,700 DEBUG HandlerThread:145 [handler.py:handle_request():146] handle_request: stop_status +2024-04-13 02:11:26,701 DEBUG SenderThread:145 [sender.py:send_request():406] send_request: stop_status +2024-04-13 02:11:26,704 DEBUG HandlerThread:145 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-13 02:11:28,082 DEBUG HandlerThread:145 [handler.py:handle_request():146] handle_request: partial_history +2024-04-13 02:11:28,085 DEBUG SenderThread:145 [sender.py:send():379] send: metric +2024-04-13 02:11:28,085 DEBUG SenderThread:145 [sender.py:send():379] send: metric +2024-04-13 02:11:28,085 DEBUG SenderThread:145 [sender.py:send():379] send: metric +2024-04-13 02:11:28,086 DEBUG SenderThread:145 [sender.py:send():379] send: metric +2024-04-13 02:11:28,086 DEBUG SenderThread:145 [sender.py:send():379] send: history +2024-04-13 02:11:28,086 DEBUG SenderThread:145 [sender.py:send_request():406] send_request: summary_record +2024-04-13 02:11:28,088 INFO SenderThread:145 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-13 02:11:28,579 INFO Thread-12 :145 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240413_021054-hjdgsc19/files/wandb-summary.json +2024-04-13 02:11:29,222 DEBUG SenderThread:145 [sender.py:send():379] send: telemetry +2024-04-13 02:11:29,223 DEBUG SenderThread:145 [sender.py:send_request():406] send_request: summary_record +2024-04-13 02:11:29,225 DEBUG HandlerThread:145 [handler.py:handle_request():146] handle_request: partial_history +2024-04-13 02:11:29,226 INFO SenderThread:145 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-13 02:11:29,226 DEBUG SenderThread:145 [sender.py:send_request():406] send_request: summary_record +2024-04-13 02:11:29,227 INFO SenderThread:145 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-13 02:11:29,227 DEBUG SenderThread:145 [sender.py:send_request():406] send_request: summary_record +2024-04-13 02:11:29,227 INFO SenderThread:145 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-13 02:11:29,228 DEBUG SenderThread:145 [sender.py:send_request():406] send_request: summary_record +2024-04-13 02:11:29,228 INFO SenderThread:145 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-13 02:11:29,228 DEBUG SenderThread:145 [sender.py:send_request():406] send_request: summary_record +2024-04-13 02:11:29,229 INFO SenderThread:145 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-13 02:11:29,229 DEBUG SenderThread:145 [sender.py:send():379] send: history +2024-04-13 02:11:29,229 DEBUG SenderThread:145 [sender.py:send_request():406] send_request: summary_record +2024-04-13 02:11:29,230 INFO SenderThread:145 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-13 02:11:29,580 INFO Thread-12 :145 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240413_021054-hjdgsc19/files/wandb-summary.json +2024-04-13 02:11:31,231 DEBUG HandlerThread:145 [handler.py:handle_request():146] handle_request: status_report