Training in progress, step 1450

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d4b34c8309c1ee9824f8fc318cf34b6c183dc01f134642fce8deeca8188da77a
+oid sha256:ee9d564107240a10dca279adc8530a248e266892b978974825d24fe992576f65
 size 485755120

wandb/debug-internal.log

@@ -1377,3 +1377,44 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-06-02 22:13:44,953 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages
 2024-06-02 22:13:45,617 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status
 2024-06-02 22:13:45,618 DEBUG   SenderThread:82 [sender.py:send_request():406] send_request: stop_status
+2024-06-02 22:13:47,675 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:13:52,676 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:13:57,676 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:13:59,953 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages
+2024-06-02 22:14:00,617 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status
+2024-06-02 22:14:00,618 DEBUG   SenderThread:82 [sender.py:send_request():406] send_request: stop_status
+2024-06-02 22:14:02,707 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:14:07,708 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:14:12,709 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:14:13,620 DEBUG   SenderThread:82 [sender.py:send():379] send: stats
+2024-06-02 22:14:14,953 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages
+2024-06-02 22:14:15,618 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status
+2024-06-02 22:14:15,618 DEBUG   SenderThread:82 [sender.py:send_request():406] send_request: stop_status
+2024-06-02 22:14:17,721 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:14:22,723 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:14:27,723 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:14:29,953 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages
+2024-06-02 22:14:30,618 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status
+2024-06-02 22:14:30,618 DEBUG   SenderThread:82 [sender.py:send_request():406] send_request: stop_status
+2024-06-02 22:14:33,684 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:14:38,685 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:14:43,621 DEBUG   SenderThread:82 [sender.py:send():379] send: stats
+2024-06-02 22:14:44,622 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:14:44,953 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages
+2024-06-02 22:14:45,618 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status
+2024-06-02 22:14:45,618 DEBUG   SenderThread:82 [sender.py:send_request():406] send_request: stop_status
+2024-06-02 22:14:49,703 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:14:54,704 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:14:59,705 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:14:59,954 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages
+2024-06-02 22:15:00,628 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status
+2024-06-02 22:15:00,628 DEBUG   SenderThread:82 [sender.py:send_request():406] send_request: stop_status
+2024-06-02 22:15:05,703 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:15:10,704 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:15:13,622 DEBUG   SenderThread:82 [sender.py:send():379] send: stats
+2024-06-02 22:15:14,956 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages
+2024-06-02 22:15:15,623 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status
+2024-06-02 22:15:15,624 DEBUG   SenderThread:82 [sender.py:send_request():406] send_request: stop_status
+2024-06-02 22:15:15,682 INFO    Thread-12 :82 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_212342-ypv2rcj7/files/output.log
+2024-06-02 22:15:16,670 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:15:21,671 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report

wandb/run-20240602_212342-ypv2rcj7/files/output.log

@@ -56,4 +56,6 @@
   warnings.warn('Was asked to gather along dimension 0, but all '
 /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.
   warnings.warn('Was asked to gather along dimension 0, but all '
+/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.
+  warnings.warn('Was asked to gather along dimension 0, but all '
 /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.

wandb/run-20240602_212342-ypv2rcj7/logs/debug-internal.log

@@ -1377,3 +1377,44 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-06-02 22:13:44,953 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages
 2024-06-02 22:13:45,617 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status
 2024-06-02 22:13:45,618 DEBUG   SenderThread:82 [sender.py:send_request():406] send_request: stop_status
+2024-06-02 22:13:47,675 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:13:52,676 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:13:57,676 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:13:59,953 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages
+2024-06-02 22:14:00,617 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status
+2024-06-02 22:14:00,618 DEBUG   SenderThread:82 [sender.py:send_request():406] send_request: stop_status
+2024-06-02 22:14:02,707 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:14:07,708 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:14:12,709 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:14:13,620 DEBUG   SenderThread:82 [sender.py:send():379] send: stats
+2024-06-02 22:14:14,953 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages
+2024-06-02 22:14:15,618 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status
+2024-06-02 22:14:15,618 DEBUG   SenderThread:82 [sender.py:send_request():406] send_request: stop_status
+2024-06-02 22:14:17,721 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:14:22,723 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:14:27,723 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:14:29,953 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages
+2024-06-02 22:14:30,618 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status
+2024-06-02 22:14:30,618 DEBUG   SenderThread:82 [sender.py:send_request():406] send_request: stop_status
+2024-06-02 22:14:33,684 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:14:38,685 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:14:43,621 DEBUG   SenderThread:82 [sender.py:send():379] send: stats
+2024-06-02 22:14:44,622 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:14:44,953 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages
+2024-06-02 22:14:45,618 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status
+2024-06-02 22:14:45,618 DEBUG   SenderThread:82 [sender.py:send_request():406] send_request: stop_status
+2024-06-02 22:14:49,703 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:14:54,704 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:14:59,705 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:14:59,954 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages
+2024-06-02 22:15:00,628 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status
+2024-06-02 22:15:00,628 DEBUG   SenderThread:82 [sender.py:send_request():406] send_request: stop_status
+2024-06-02 22:15:05,703 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:15:10,704 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:15:13,622 DEBUG   SenderThread:82 [sender.py:send():379] send: stats
+2024-06-02 22:15:14,956 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: internal_messages
+2024-06-02 22:15:15,623 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: stop_status
+2024-06-02 22:15:15,624 DEBUG   SenderThread:82 [sender.py:send_request():406] send_request: stop_status
+2024-06-02 22:15:15,682 INFO    Thread-12 :82 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240602_212342-ypv2rcj7/files/output.log
+2024-06-02 22:15:16,670 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report
+2024-06-02 22:15:21,671 DEBUG   HandlerThread:82 [handler.py:handle_request():146] handle_request: status_report