sharukat commited on
Commit
2e496ae
·
verified ·
1 Parent(s): 0e9fff5

Training in progress, epoch 5

Browse files
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:184f2bb730d40284f45451db2d209e9f5cc74e6b4657c771fb9eb210675c8cc0
3
  size 502675828
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f92cf0b63337107efe8f72177c57bb836f176cfa48fc5d52a3213683c7fcb2e3
3
  size 502675828
runs/Mar06_15-06-49_41759fa8e6ad/events.out.tfevents.1709737609.41759fa8e6ad.34.4 CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:987af32ac547c8583921bb99d3d3a0ce9deb1ac55f15962995271473124e96d4
3
- size 7850
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3d48f1729e2ef03bdaceb8ff419b66a5cc257fe32d744152ebd794b459fb6766
3
+ size 8533
wandb/debug-internal.log CHANGED
@@ -803,3 +803,33 @@
803
  2024-03-06 15:09:11,804 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
804
  2024-03-06 15:09:13,724 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
805
  2024-03-06 15:09:15,358 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
803
  2024-03-06 15:09:11,804 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
804
  2024-03-06 15:09:13,724 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
805
  2024-03-06 15:09:15,358 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
806
+ 2024-03-06 15:09:16,805 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
807
+ 2024-03-06 15:09:18,734 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
808
+ 2024-03-06 15:09:20,359 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
809
+ 2024-03-06 15:09:21,806 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
810
+ 2024-03-06 15:09:23,735 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
811
+ 2024-03-06 15:09:25,360 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
812
+ 2024-03-06 15:09:26,801 DEBUG SenderThread:137 [sender.py:send():382] send: stats
813
+ 2024-03-06 15:09:27,802 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
814
+ 2024-03-06 15:09:28,675 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
815
+ 2024-03-06 15:09:28,676 DEBUG SenderThread:137 [sender.py:send():382] send: history
816
+ 2024-03-06 15:09:28,676 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record
817
+ 2024-03-06 15:09:28,677 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
818
+ 2024-03-06 15:09:28,736 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
819
+ 2024-03-06 15:09:29,091 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
820
+ 2024-03-06 15:09:29,792 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
821
+ 2024-03-06 15:09:29,793 DEBUG SenderThread:137 [sender.py:send():382] send: history
822
+ 2024-03-06 15:09:29,793 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record
823
+ 2024-03-06 15:09:29,794 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
824
+ 2024-03-06 15:09:30,092 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
825
+ 2024-03-06 15:09:30,361 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
826
+ 2024-03-06 15:09:32,093 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log
827
+ 2024-03-06 15:09:33,754 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
828
+ 2024-03-06 15:09:33,795 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
829
+ 2024-03-06 15:09:35,362 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
830
+ 2024-03-06 15:09:38,755 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
831
+ 2024-03-06 15:09:38,796 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
832
+ 2024-03-06 15:09:40,363 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
833
+ 2024-03-06 15:09:43,756 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
834
+ 2024-03-06 15:09:43,797 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
835
+ 2024-03-06 15:09:45,364 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
wandb/run-20240306_145424-trm7fvg4/logs/debug-internal.log CHANGED
@@ -854,3 +854,33 @@ wandb.errors.AuthenticationError: The API key you provided is either invalid or
854
  2024-03-06 15:09:11,804 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
855
  2024-03-06 15:09:13,724 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
856
  2024-03-06 15:09:15,358 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
854
  2024-03-06 15:09:11,804 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
855
  2024-03-06 15:09:13,724 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
856
  2024-03-06 15:09:15,358 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
857
+ 2024-03-06 15:09:16,805 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
858
+ 2024-03-06 15:09:18,734 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
859
+ 2024-03-06 15:09:20,359 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
860
+ 2024-03-06 15:09:21,806 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
861
+ 2024-03-06 15:09:23,735 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
862
+ 2024-03-06 15:09:25,360 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
863
+ 2024-03-06 15:09:26,801 DEBUG SenderThread:137 [sender.py:send():382] send: stats
864
+ 2024-03-06 15:09:27,802 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
865
+ 2024-03-06 15:09:28,675 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
866
+ 2024-03-06 15:09:28,676 DEBUG SenderThread:137 [sender.py:send():382] send: history
867
+ 2024-03-06 15:09:28,676 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record
868
+ 2024-03-06 15:09:28,677 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
869
+ 2024-03-06 15:09:28,736 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
870
+ 2024-03-06 15:09:29,091 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
871
+ 2024-03-06 15:09:29,792 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
872
+ 2024-03-06 15:09:29,793 DEBUG SenderThread:137 [sender.py:send():382] send: history
873
+ 2024-03-06 15:09:29,793 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record
874
+ 2024-03-06 15:09:29,794 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
875
+ 2024-03-06 15:09:30,092 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
876
+ 2024-03-06 15:09:30,361 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
877
+ 2024-03-06 15:09:32,093 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log
878
+ 2024-03-06 15:09:33,754 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
879
+ 2024-03-06 15:09:33,795 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
880
+ 2024-03-06 15:09:35,362 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
881
+ 2024-03-06 15:09:38,755 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
882
+ 2024-03-06 15:09:38,796 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
883
+ 2024-03-06 15:09:40,363 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
884
+ 2024-03-06 15:09:43,756 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
885
+ 2024-03-06 15:09:43,797 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
886
+ 2024-03-06 15:09:45,364 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
wandb/run-20240306_145455-h1uv5tyi/files/output.log CHANGED
@@ -44,3 +44,5 @@ Checkpoint destination directory /kaggle/working/checkpoint-62 already exists an
44
  _warn_prf(average, modifier, msg_start, len(result))
45
  /opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
46
  _warn_prf(average, modifier, msg_start, len(result))
 
 
 
44
  _warn_prf(average, modifier, msg_start, len(result))
45
  /opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
46
  _warn_prf(average, modifier, msg_start, len(result))
47
+ Checkpoint destination directory /kaggle/working/checkpoint-124 already exists and is non-empty. Saving will proceed but saved results may be invalid.
48
+ /opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json CHANGED
@@ -1 +1 @@
1
- {"train/loss": 1.652, "train/grad_norm": 5.863879680633545, "train/learning_rate": 1.8e-05, "train/epoch": 4.0, "train/global_step": 124, "_timestamp": 1709737736.6036682, "_runtime": 840.8849112987518, "_step": 33, "eval/loss": 1.7384552955627441, "eval/accuracy": 0.34545454545454546, "eval/precision": 0.19107438016528924, "eval/recall": 0.34545454545454546, "eval/f1": 0.2349731077003804, "eval/runtime": 1.1115, "eval/samples_per_second": 49.484, "eval/steps_per_second": 3.599, "train/train_runtime": 237.4526, "train/train_samples_per_second": 10.339, "train/train_steps_per_second": 1.306, "train/total_flos": 645966638976000.0, "train/train_loss": 1.7031736066264491}
 
1
+ {"train/loss": 1.6049, "train/grad_norm": NaN, "train/learning_rate": 1.5096774193548387e-05, "train/epoch": 5.0, "train/global_step": 155, "_timestamp": 1709737769.791673, "_runtime": 874.0729160308838, "_step": 35, "eval/loss": 1.730015516281128, "eval/accuracy": 0.36363636363636365, "eval/precision": 0.19892561983471074, "eval/recall": 0.36363636363636365, "eval/f1": 0.2459399186671914, "eval/runtime": 1.1125, "eval/samples_per_second": 49.439, "eval/steps_per_second": 3.596, "train/train_runtime": 237.4526, "train/train_samples_per_second": 10.339, "train/train_steps_per_second": 1.306, "train/total_flos": 645966638976000.0, "train/train_loss": 1.7031736066264491}
wandb/run-20240306_145455-h1uv5tyi/logs/debug-internal.log CHANGED
@@ -803,3 +803,33 @@
803
  2024-03-06 15:09:11,804 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
804
  2024-03-06 15:09:13,724 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
805
  2024-03-06 15:09:15,358 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
803
  2024-03-06 15:09:11,804 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
804
  2024-03-06 15:09:13,724 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
805
  2024-03-06 15:09:15,358 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
806
+ 2024-03-06 15:09:16,805 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
807
+ 2024-03-06 15:09:18,734 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
808
+ 2024-03-06 15:09:20,359 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
809
+ 2024-03-06 15:09:21,806 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
810
+ 2024-03-06 15:09:23,735 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
811
+ 2024-03-06 15:09:25,360 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
812
+ 2024-03-06 15:09:26,801 DEBUG SenderThread:137 [sender.py:send():382] send: stats
813
+ 2024-03-06 15:09:27,802 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
814
+ 2024-03-06 15:09:28,675 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
815
+ 2024-03-06 15:09:28,676 DEBUG SenderThread:137 [sender.py:send():382] send: history
816
+ 2024-03-06 15:09:28,676 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record
817
+ 2024-03-06 15:09:28,677 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
818
+ 2024-03-06 15:09:28,736 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
819
+ 2024-03-06 15:09:29,091 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
820
+ 2024-03-06 15:09:29,792 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
821
+ 2024-03-06 15:09:29,793 DEBUG SenderThread:137 [sender.py:send():382] send: history
822
+ 2024-03-06 15:09:29,793 DEBUG SenderThread:137 [sender.py:send_request():409] send_request: summary_record
823
+ 2024-03-06 15:09:29,794 INFO SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
824
+ 2024-03-06 15:09:30,092 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
825
+ 2024-03-06 15:09:30,361 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
826
+ 2024-03-06 15:09:32,093 INFO Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log
827
+ 2024-03-06 15:09:33,754 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
828
+ 2024-03-06 15:09:33,795 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
829
+ 2024-03-06 15:09:35,362 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
830
+ 2024-03-06 15:09:38,755 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
831
+ 2024-03-06 15:09:38,796 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
832
+ 2024-03-06 15:09:40,363 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
833
+ 2024-03-06 15:09:43,756 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
834
+ 2024-03-06 15:09:43,797 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
835
+ 2024-03-06 15:09:45,364 DEBUG HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report