Training in progress, epoch 4

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+wandb/run-20240201_175850-i93q0p12/run-i93q0p12.wandb filter=lfs diff=lfs merge=lfs -text

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b36e69ffbe5ae42f8478ebfbc7c07012a026dc6fbd9407f024781cecd7b9d168
+oid sha256:f328f655a6c8b38df202a4de4e669604f27ad31b0420d0d6742110d15af58f36
 size 3141646744

runs/Feb01_17-58-13_ip-26-0-165-24/events.out.tfevents.1706810328.ip-26-0-165-24.239318.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6335f8b18c2d252b1614b2ce15bdaae4fa5dfa041da34f64c37eeab8379c14b0
-size 9006
+oid sha256:0603591350ae6d59bd20161e6fbc71c11f9404d1bf38d86d0e0cf98885ef67f2
+size 22693

wandb/run-20240201_175850-i93q0p12/files/output.log

@@ -356,3 +356,1010 @@
 
  21%|████████████████▌                                                             | 290/1365 [18:14<1:04:59,  3.63s/it]
 
+
+
+
+
+
+
+
+
+
+ 22%|█████████████████▏                                                            | 300/1365 [18:50<1:03:46,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 23%|█████████████████▋                                                            | 310/1365 [19:25<1:03:01,  3.58s/it]
+
+
+
+
+
+
+
+
+
+
+ 23%|██████████████████▎                                                           | 320/1365 [20:01<1:02:25,  3.58s/it]
+
+
+
+
+
+
+
+
+
+
+ 24%|██████████████████▊                                                           | 330/1365 [20:37<1:01:50,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 25%|███████████████████▍                                                          | 340/1365 [21:13<1:01:12,  3.58s/it]
+
+
+
+
+
+
+
+
+
+
+ 26%|████████████████████                                                          | 350/1365 [21:49<1:00:35,  3.58s/it]
+
+
+
+
+
+
+
+
+
+
+ 26%|█████████████████████                                                           | 360/1365 [22:24<59:59,  3.58s/it]
+
+
+
+
+
+
+
+
+
+
+ 27%|█████████████████████▋                                                          | 370/1365 [23:00<59:23,  3.58s/it]
+
+
+
+
+
+
+
+
+
+ 28%|██████████████████████▏                                                         | 379/1365 [23:33<58:51,  3.58s/it]
+
+
+
+
+
+
+
+
+
+
+ 28%|██████████████████████▊                                                         | 389/1365 [24:08<58:15,  3.58s/it]
+
+
+
+
+
+
+
+
+
+
+
+ 29%|███████████████████████▍                                                        | 400/1365 [24:48<57:37,  3.58s/it]
+
+
+
+
+
+
+
+
+
+
+ 30%|████████████████████████                                                        | 410/1365 [25:24<56:59,  3.58s/it]
+
+
+
+
+
+
+
+
+
+
+ 31%|████████████████████████▌                                                       | 420/1365 [25:59<56:22,  3.58s/it]
+
+
+
+
+
+
+
+
+
+
+ 32%|█████████████████████████▏                                                      | 430/1365 [26:36<57:28,  3.69s/it]
+
+
+
+
+
+
+
+
+
+
+ 32%|█████████████████████████▊                                                      | 440/1365 [27:13<56:44,  3.68s/it]
+
+
+
+
+
+
+
+
+
+
+ 33%|██████████████████████████▎                                                     | 450/1365 [27:49<55:38,  3.65s/it]
+
+
+
+
+
+
+
+
+
+
+ 34%|██████████████████████████▉                                                     | 460/1365 [28:26<54:37,  3.62s/it]
+
+
+
+
+
+
+
+
+
+
+ 34%|███████████████████████████▌                                                    | 470/1365 [29:02<53:57,  3.62s/it]
+
+
+
+
+
+
+
+
+
+
+ 35%|████████████████████████████▏                                                   | 480/1365 [29:38<53:18,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+ 36%|████████████████████████████▋                                                   | 490/1365 [30:14<52:50,  3.62s/it]
+
+
+
+
+
+
+
+
+
+
+ 37%|█████████████████████████████▎                                                  | 500/1365 [30:51<52:27,  3.64s/it]
+
+
+
+
+
+
+
+
+
+
+ 37%|█████████████████████████████▉                                                  | 510/1365 [31:27<51:33,  3.62s/it]
+
+
+
+
+
+
+
+
+
+
+ 38%|██████████████████████████████▍                                                 | 520/1365 [32:03<50:52,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+ 39%|███████████████████████████████                                                 | 530/1365 [32:39<50:11,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+ 40%|███████████████████████████████▋                                                | 540/1365 [33:15<49:33,  3.60s/it]
+
+
+
+
+
+ 40%|████████████████████████████████                                                | 546/1365 [33:37<48:43,  3.57s/it][INFO|trainer.py:3166] 2024-02-01 18:32:32,716 >> ***** Running Evaluation *****
+[INFO|trainer.py:3168] 2024-02-01 18:32:32,716 >>   Num examples = 15431
+[INFO|trainer.py:3171] 2024-02-01 18:32:32,716 >>   Batch size = 32
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 95%|██████████████████████████████████████████████████████████████████████████████▉    | 58/61 [00:28<00:01,  1.99it/s]
+ 40%|████████████████████████████████                                                | 546/1365 [34:07<48:43,  3.57s/it][INFO|trainer.py:2889] 2024-02-01 18:33:04,367 >> Saving model checkpoint to ./tmp-checkpoint-546
+[INFO|configuration_utils.py:483] 2024-02-01 18:33:04,370 >> Configuration saved in ./tmp-checkpoint-546/config.json
+[INFO|configuration_utils.py:594] 2024-02-01 18:33:04,373 >> Configuration saved in ./tmp-checkpoint-546/generation_config.json
+[INFO|modeling_utils.py:2382] 2024-02-01 18:33:07,531 >> Model weights saved in ./tmp-checkpoint-546/pytorch_model.bin
+[INFO|tokenization_utils_base.py:2432] 2024-02-01 18:33:07,550 >> tokenizer config file saved in ./tmp-checkpoint-546/tokenizer_config.json
+[INFO|tokenization_utils_base.py:2441] 2024-02-01 18:33:07,552 >> Special tokens file saved in ./tmp-checkpoint-546/special_tokens_map.json
+/fsx/sanchit/miniconda3/envs/venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1879: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.
+  warnings.warn(
+[2024-02-01 18:33:07,612] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step546 is about to be saved!
+[2024-02-01 18:33:07,619] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./tmp-checkpoint-546/global_step546/zero_pp_rank_0_mp_rank_00_model_states.pt
+[2024-02-01 18:33:07,620] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./tmp-checkpoint-546/global_step546/zero_pp_rank_0_mp_rank_00_model_states.pt...
+[2024-02-01 18:33:07,625] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./tmp-checkpoint-546/global_step546/zero_pp_rank_0_mp_rank_00_model_states.pt.
+[2024-02-01 18:33:07,633] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./tmp-checkpoint-546/global_step546/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2024-02-01 18:33:11,463] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./tmp-checkpoint-546/global_step546/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2024-02-01 18:33:11,471] [INFO] [engine.py:3393:_save_zero_checkpoint] zero checkpoint saved ./tmp-checkpoint-546/global_step546/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2024-02-01 18:33:11,659] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step546 is ready now!
+[INFO|tokenization_utils_base.py:2432] 2024-02-01 18:33:14,019 >> tokenizer config file saved in ./tokenizer_config.json
+[INFO|tokenization_utils_base.py:2441] 2024-02-01 18:33:14,021 >> Special tokens file saved in ./special_tokens_map.json
+[INFO|trainer.py:2979] 2024-02-01 18:33:14,053 >> Deleting older checkpoint [checkpoint-273] due to args.save_total_limit
+
+
+
+ 40%|███████████████████████████████▍                                              | 550/1365 [34:33<1:46:53,  7.87s/it]
+
+
+
+
+
+
+
+
+
+
+ 41%|████████████████████████████████▊                                               | 560/1365 [35:09<49:51,  3.72s/it]
+
+
+
+
+
+
+
+
+
+
+ 42%|█████████████████████████████████▍                                              | 570/1365 [35:45<47:37,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 42%|█████████████████████████████████▉                                              | 580/1365 [36:21<46:58,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 43%|██████████████████████████████████▌                                             | 590/1365 [36:57<46:21,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 44%|███████████████████████████████████▏                                            | 600/1365 [37:32<45:44,  3.59s/it]
+
+
+
+
+
+
+
+
+
+ 45%|███████████████████████████████████▋                                            | 609/1365 [38:05<45:14,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 45%|████████████████████████████████████▎                                           | 619/1365 [38:41<44:40,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+
+ 46%|████████████████████████████████████▉                                           | 630/1365 [39:20<43:58,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 47%|█████████████████████████████████████▌                                          | 640/1365 [39:56<43:26,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 48%|██████████████████████████████████████                                          | 650/1365 [40:32<42:50,  3.60s/it]
+
+
+
+
+
+
+
+
+
+
+ 48%|██████████████████████████████████████▋                                         | 660/1365 [41:08<42:13,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 49%|███████████████████████████████████████▎                                        | 670/1365 [41:44<41:36,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 50%|███████████████████████████████████████▊                                        | 680/1365 [42:20<41:00,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 51%|████████████████████████████████████████▍                                       | 690/1365 [42:56<40:27,  3.60s/it]
+
+
+
+
+
+
+
+
+
+
+ 51%|█████████████████████████████████████████                                       | 700/1365 [43:32<39:48,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 52%|█████████████████████████████████████████▌                                      | 710/1365 [44:08<39:42,  3.64s/it]
+
+
+
+
+
+
+
+
+
+ 53%|██████████████████████████████████████████▏                                     | 719/1365 [44:41<38:56,  3.62s/it]
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████████████████████████████████████▊                                     | 730/1365 [45:20<38:11,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+ 54%|███████████████████████████████████████████▎                                    | 740/1365 [45:57<37:33,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+ 55%|███████████████████████████████████████████▉                                    | 750/1365 [46:33<36:57,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+ 56%|████████████████████████████████████████████▌                                   | 760/1365 [47:09<36:23,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+ 56%|█████████████████████████████████████████████▏                                  | 770/1365 [47:45<35:44,  3.60s/it]
+
+
+
+
+
+
+
+
+
+
+ 57%|█████████████████████████████████████████████▋                                  | 780/1365 [48:21<35:09,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+ 58%|██████████████████████████████████████████████▎                                 | 790/1365 [48:57<34:34,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+ 59%|██████████████████████████████████████████████▉                                 | 800/1365 [49:33<33:56,  3.60s/it]
+
+
+
+
+
+
+
+
+
+
+ 59%|███████████████████████████████████████████████▍                                | 810/1365 [50:09<33:19,  3.60s/it]
+
+
+
+
+
+
+
+
+ 60%|████████████████████████████████████████████████                                | 819/1365 [50:41<32:27,  3.57s/it][INFO|trainer.py:3166] 2024-02-01 18:49:37,123 >> ***** Running Evaluation *****
+[INFO|trainer.py:3168] 2024-02-01 18:49:37,124 >>   Num examples = 15431
+[INFO|trainer.py:3171] 2024-02-01 18:49:37,124 >>   Batch size = 32
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 95%|██████████████████████████████████████████████████████████████████████████████▉    | 58/61 [00:28<00:01,  2.00it/s]
+ 60%|█████████���██████████████████████████████████████                                | 819/1365 [51:12<32:27,  3.57s/it][INFO|trainer.py:2889] 2024-02-01 18:50:08,864 >> Saving model checkpoint to ./tmp-checkpoint-819
+[INFO|configuration_utils.py:483] 2024-02-01 18:50:08,868 >> Configuration saved in ./tmp-checkpoint-819/config.json
+[INFO|configuration_utils.py:594] 2024-02-01 18:50:08,870 >> Configuration saved in ./tmp-checkpoint-819/generation_config.json
+[INFO|modeling_utils.py:2382] 2024-02-01 18:50:11,947 >> Model weights saved in ./tmp-checkpoint-819/pytorch_model.bin
+[INFO|tokenization_utils_base.py:2432] 2024-02-01 18:50:11,966 >> tokenizer config file saved in ./tmp-checkpoint-819/tokenizer_config.json
+[INFO|tokenization_utils_base.py:2441] 2024-02-01 18:50:11,968 >> Special tokens file saved in ./tmp-checkpoint-819/special_tokens_map.json
+/fsx/sanchit/miniconda3/envs/venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1879: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.
+  warnings.warn(
+[2024-02-01 18:50:12,033] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step819 is about to be saved!
+[2024-02-01 18:50:12,045] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./tmp-checkpoint-819/global_step819/zero_pp_rank_0_mp_rank_00_model_states.pt
+[2024-02-01 18:50:12,045] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./tmp-checkpoint-819/global_step819/zero_pp_rank_0_mp_rank_00_model_states.pt...
+[2024-02-01 18:50:12,049] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./tmp-checkpoint-819/global_step819/zero_pp_rank_0_mp_rank_00_model_states.pt.
+[2024-02-01 18:50:12,059] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./tmp-checkpoint-819/global_step819/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2024-02-01 18:50:15,835] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./tmp-checkpoint-819/global_step819/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2024-02-01 18:50:15,843] [INFO] [engine.py:3393:_save_zero_checkpoint] zero checkpoint saved ./tmp-checkpoint-819/global_step819/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2024-02-01 18:50:16,356] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step819 is ready now!
+[INFO|tokenization_utils_base.py:2432] 2024-02-01 18:50:18,699 >> tokenizer config file saved in ./tokenizer_config.json
+[INFO|tokenization_utils_base.py:2441] 2024-02-01 18:50:18,701 >> Special tokens file saved in ./special_tokens_map.json
+[INFO|trainer.py:2979] 2024-02-01 18:50:18,733 >> Deleting older checkpoint [checkpoint-546] due to args.save_total_limit
+ 60%|██████████████████████████████████████████████▊                               | 820/1365 [51:27<2:26:38, 16.14s/it]
+
+
+
+
+
+
+
+
+
+
+ 61%|████████████████████████████████████████████████▋                               | 830/1365 [52:03<35:06,  3.94s/it]
+
+
+
+
+
+
+
+
+
+
+ 62%|█████████████████████████████████████████████████▏                              | 840/1365 [52:38<31:23,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 62%|█████████████████████████████████████████████████▊                              | 850/1365 [53:14<30:45,  3.58s/it]
+
+
+
+
+
+
+
+
+
+
+ 63%|██████████████████████████████████████████████████▍                             | 860/1365 [53:50<30:11,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 64%|██████████████████████████████████████████████████▉                             | 870/1365 [54:26<29:35,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 64%|███████████████████████████████████████████████████▌                            | 880/1365 [55:02<28:58,  3.58s/it]
+
+
+
+
+
+
+
+
+
+
+ 65%|████████████████████████████████████████████████████▏                           | 890/1365 [55:38<28:21,  3.58s/it]
+
+
+
+
+
+
+
+
+
+
+ 66%|████████████████████████████████████████████████████▋                           | 900/1365 [56:13<27:47,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 67%|█████████████████████████████████████████████████████▎                          | 910/1365 [56:49<27:11,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 67%|█████████████████████████████████████████████████████▉                          | 920/1365 [57:25<26:35,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 68%|██████████████████████████████████████████████████████▌                         | 930/1365 [58:02<26:30,  3.66s/it]
+
+
+
+
+
+
+
+
+
+
+ 69%|███████████████████████████████████████████████████████                         | 940/1365 [58:37<25:23,  3.58s/it]
+
+
+
+
+
+
+
+
+
+
+ 70%|███████████████████████████████████████████████████████▋                        | 950/1365 [59:13<24:49,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 70%|████████████████████████████████████████████████████████▎                       | 960/1365 [59:49<24:11,  3.58s/it]
+
+
+
+
+
+
+
+
+
+
+ 71%|███████████████████████████████████████████████████████▍                      | 970/1365 [1:00:25<23:37,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 72%|████████████████████████████████████████████████████████                      | 980/1365 [1:01:02<23:42,  3.69s/it]
+
+
+
+
+
+
+
+
+
+
+ 73%|████████████████████████████████████████████████████████▌                     | 990/1365 [1:01:38<22:46,  3.65s/it]
+
+
+
+
+
+
+
+
+
+
+ 73%|████████████████████████████████████████████████████████▍                    | 1000/1365 [1:02:15<22:04,  3.63s/it]
+
+
+
+
+
+
+
+
+
+
+ 74%|████████████████████████████████████████████████████████▉                    | 1010/1365 [1:02:51<21:25,  3.62s/it]
+
+
+
+
+
+
+
+
+
+
+ 75%|█████████████████████████████████████████████████████████▌                   | 1020/1365 [1:03:27<20:46,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+ 75%|██████████████████████████████████████████████████████████                   | 1030/1365 [1:04:03<20:08,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+ 76%|██████████████████████████████████████████████████████████▋                  | 1040/1365 [1:04:39<19:33,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+ 77%|███████████████████████████████████████████████████████████▏                 | 1050/1365 [1:05:15<18:57,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+ 78%|███████████████████████████████████████████████████████████▊                 | 1060/1365 [1:05:52<18:20,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+ 78%|████████████████████████████████████████████████████████████▎                | 1070/1365 [1:06:28<17:43,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+ 79%|████████████████████████████████████████████████████████████▉                | 1080/1365 [1:07:04<17:08,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+ 80%|█████████████████████████████████████████████████████████████▍               | 1090/1365 [1:07:40<16:31,  3.61s/it]
+
+ 80%|█████████████████████████████████████████████████████████████▌               | 1092/1365 [1:07:47<16:14,  3.57s/it][INFO|trainer.py:3166] 2024-02-01 19:06:42,792 >> ***** Running Evaluation *****
+[INFO|trainer.py:3168] 2024-02-01 19:06:42,792 >>   Num examples = 15431
+[INFO|trainer.py:3171] 2024-02-01 19:06:42,792 >>   Batch size = 32
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+[INFO|configuration_utils.py:483] 2024-02-01 19:07:14,404 >> Configuration saved in ./tmp-checkpoint-1092/config.json
+[INFO|configuration_utils.py:594] 2024-02-01 19:07:14,406 >> Configuration saved in ./tmp-checkpoint-1092/generation_config.json
+{'eval_loss': 2.1637322902679443, 'eval_runtime': 30.5067, 'eval_samples_per_second': 505.824, 'eval_steps_per_second': 2.0, 'epoch': 4.0}
+[INFO|modeling_utils.py:2382] 2024-02-01 19:07:17,532 >> Model weights saved in ./tmp-checkpoint-1092/pytorch_model.bin
+[INFO|tokenization_utils_base.py:2432] 2024-02-01 19:07:17,551 >> tokenizer config file saved in ./tmp-checkpoint-1092/tokenizer_config.json
+[INFO|tokenization_utils_base.py:2441] 2024-02-01 19:07:17,554 >> Special tokens file saved in ./tmp-checkpoint-1092/special_tokens_map.json
+/fsx/sanchit/miniconda3/envs/venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1879: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.
+  warnings.warn(
+[2024-02-01 19:07:17,618] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step1092 is about to be saved!
+[2024-02-01 19:07:17,624] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./tmp-checkpoint-1092/global_step1092/zero_pp_rank_0_mp_rank_00_model_states.pt
+[2024-02-01 19:07:17,624] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./tmp-checkpoint-1092/global_step1092/zero_pp_rank_0_mp_rank_00_model_states.pt...
+[2024-02-01 19:07:17,629] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./tmp-checkpoint-1092/global_step1092/zero_pp_rank_0_mp_rank_00_model_states.pt.
+[2024-02-01 19:07:17,637] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./tmp-checkpoint-1092/global_step1092/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2024-02-01 19:07:21,468] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./tmp-checkpoint-1092/global_step1092/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2024-02-01 19:07:21,476] [INFO] [engine.py:3393:_save_zero_checkpoint] zero checkpoint saved ./tmp-checkpoint-1092/global_step1092/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2024-02-01 19:07:21,809] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step1092 is ready now!
+[INFO|tokenization_utils_base.py:2432] 2024-02-01 19:07:24,145 >> tokenizer config file saved in ./tokenizer_config.json
+[INFO|tokenization_utils_base.py:2441] 2024-02-01 19:07:24,147 >> Special tokens file saved in ./special_tokens_map.json
+[INFO|trainer.py:2979] 2024-02-01 19:07:24,178 >> Deleting older checkpoint [checkpoint-819] due to args.save_total_limit
+
+
+
+
+
+
+
+ 81%|██████████████████████████████████████████████████████████████               | 1100/1365 [1:08:57<20:25,  4.62s/it]
+
+
+
+
+
+
+
+
+
+
+ 81%|██████████████████████████████████████████████████████████████▌              | 1110/1365 [1:09:33<15:23,  3.62s/it]

wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 4.6222, "train/learning_rate": 0.00028865524291388006, "train/epoch": 1.06, "train/global_step": 290, "_timestamp": 1706811429.4937887, "_runtime": 1099.4098217487335, "_step": 30, "eval/loss": 4.737916946411133, "eval/runtime": 30.7851, "eval/samples_per_second": 501.248, "eval/steps_per_second": 1.981}
+{"train/loss": 2.0771, "train/learning_rate": 3.080259523093675e-05, "train/epoch": 4.07, "train/global_step": 1110, "_timestamp": 1706814509.1468732, "_runtime": 4179.062906265259, "_step": 115, "eval/loss": 2.1637322902679443, "eval/runtime": 30.5067, "eval/samples_per_second": 505.824, "eval/steps_per_second": 2.0}