Training in progress, step 2500, checkpoint

checkpoint-2500/adapter_config.json

@@ -23,12 +23,12 @@
   "revision": null,
   "target_modules": [
     "q_proj",
-    "v_proj",
-    "k_proj",
-    "up_proj",
     "down_proj",
+    "k_proj",
     "o_proj",
-    "gate_proj"
+    "v_proj",
+    "gate_proj",
+    "up_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

checkpoint-2500/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:37951f6cc65c729a9ce2d54fa10f91d8e418166e175fff1508de86b2d4b400d0
+oid sha256:82971030ddbc8ed4dea42524dab0554d2893743b5a2990dd8a1fddf90f2fe309
 size 2460946960

checkpoint-2500/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:166a584bfd648aa5576f8a65c489b7077c355f499c21b52fd73b2f9758089a27
-size 1831554874
+oid sha256:81f45d14a05279c958d566d8b50abbacc972be214b58e8d697a5a110137eebf6
+size 1831543994

checkpoint-2500/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d8b5d4df779ffc182f1606eda9f6fbf986ddc8ee2ee85118303e388d7ac45064
+oid sha256:dd5bc3b5f58a5ab5b07a199a5cee4cb2ef460e0a2ac7342fe01496ddb803cd41
 size 14308

checkpoint-2500/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0f05609aa2976f6f4fb0e4b3b4f462685632ae1d6ed868b587dfcae8bb1f6028
+oid sha256:13712619cb3399d895b0657a6d715ac5aaec0819cc89786eb755488b638f7dc0
 size 1064

checkpoint-2500/special_tokens_map.json

@@ -1,20 +1,4 @@
 {
-  "additional_special_tokens": [
-    {
-      "content": "[SQL]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "[/SQL]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    }
-  ],
   "bos_token": {
     "content": "<|endoftext|>",
     "lstrip": false,

checkpoint-2500/tokenizer.json

@@ -437,8 +437,8 @@
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
-      "special": true
+      "normalized": true,
+      "special": false
     },
     {
       "id": 50300,
@@ -446,8 +446,8 @@
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
-      "special": true
+      "normalized": true,
+      "special": false
     }
   ],
   "normalizer": {

checkpoint-2500/tokenizer_config.json

@@ -382,30 +382,30 @@
     "50299": {
       "content": "[SQL]",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
-      "special": true
+      "special": false
     },
     "50300": {
       "content": "[/SQL]",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
-      "special": true
+      "special": false
     }
   },
-  "additional_special_tokens": [
-    "[SQL]",
-    "[/SQL]"
-  ],
   "bos_token": "<|endoftext|>",
   "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = 'You are a helpful assistant.' %}{% endif %}{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in loop_messages %}{% if loop.index0 == 0 %}{{'<|im_start|>system\n' + system_message + '<|im_end|>\n'}}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|endoftext|>",
+  "max_length": 1024,
   "model_max_length": 4096,
   "pad_token": "<|endoftext|>",
+  "stride": 0,
   "tokenizer_class": "GPTNeoXTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "<|endoftext|>"
 }

checkpoint-2500/trainer_state.json

@@ -1,7 +1,7 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.5461496450027308,
+  "epoch": 2.184796854521625,
   "eval_steps": 500,
   "global_step": 2500,
   "is_hyper_param_search": false,
@@ -9,88 +9,88 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.10922992900054615,
-      "grad_norm": 0.9561710357666016,
-      "learning_rate": 1.4564520827264783e-05,
-      "loss": 0.7634,
+      "epoch": 0.46,
+      "grad_norm": 0.36198097467422485,
+      "learning_rate": 6.0716454159077105e-05,
+      "loss": 0.4455,
       "step": 500
     },
     {
-      "epoch": 0.10922992900054615,
-      "eval_loss": 0.3866455852985382,
-      "eval_runtime": 53.8542,
-      "eval_samples_per_second": 1.727,
-      "eval_steps_per_second": 1.727,
+      "epoch": 0.46,
+      "eval_loss": 0.6183120608329773,
+      "eval_runtime": 195.3588,
+      "eval_samples_per_second": 2.37,
+      "eval_steps_per_second": 2.37,
       "step": 500
     },
     {
-      "epoch": 0.2184598580010923,
-      "grad_norm": 0.6381384134292603,
-      "learning_rate": 2.9129041654529566e-05,
-      "loss": 0.2265,
+      "epoch": 0.91,
+      "grad_norm": 0.1898297369480133,
+      "learning_rate": 0.00012143290831815421,
+      "loss": 0.0995,
       "step": 1000
     },
     {
-      "epoch": 0.2184598580010923,
-      "eval_loss": 0.28045815229415894,
-      "eval_runtime": 53.8781,
-      "eval_samples_per_second": 1.726,
-      "eval_steps_per_second": 1.726,
+      "epoch": 0.91,
+      "eval_loss": 0.6644699573516846,
+      "eval_runtime": 195.5381,
+      "eval_samples_per_second": 2.368,
+      "eval_steps_per_second": 2.368,
       "step": 1000
     },
     {
-      "epoch": 0.3276897870016384,
-      "grad_norm": 0.80877685546875,
-      "learning_rate": 4.3693562481794354e-05,
-      "loss": 0.13,
+      "epoch": 1.37,
+      "grad_norm": 0.25010770559310913,
+      "learning_rate": 0.00018214936247723133,
+      "loss": 0.0858,
       "step": 1500
     },
     {
-      "epoch": 0.3276897870016384,
-      "eval_loss": 0.24824748933315277,
-      "eval_runtime": 53.7906,
-      "eval_samples_per_second": 1.729,
-      "eval_steps_per_second": 1.729,
+      "epoch": 1.37,
+      "eval_loss": 0.6735700368881226,
+      "eval_runtime": 195.292,
+      "eval_samples_per_second": 2.371,
+      "eval_steps_per_second": 2.371,
       "step": 1500
     },
     {
-      "epoch": 0.4369197160021846,
-      "grad_norm": 0.7288264632225037,
-      "learning_rate": 5.8228954267404603e-05,
-      "loss": 0.1067,
+      "epoch": 1.7479248580166011,
+      "grad_norm": 0.12149681895971298,
+      "learning_rate": 0.00019897628145050826,
+      "loss": 0.0836,
       "step": 2000
     },
     {
-      "epoch": 0.4369197160021846,
-      "eval_loss": 0.24914385378360748,
-      "eval_runtime": 53.8469,
-      "eval_samples_per_second": 1.727,
-      "eval_steps_per_second": 1.727,
+      "epoch": 1.7479248580166011,
+      "eval_loss": 0.2921118438243866,
+      "eval_runtime": 49.9169,
+      "eval_samples_per_second": 1.863,
+      "eval_steps_per_second": 1.863,
       "step": 2000
     },
     {
-      "epoch": 0.5461496450027308,
-      "grad_norm": 1.1100046634674072,
-      "learning_rate": 7.27934750946694e-05,
-      "loss": 0.1018,
+      "epoch": 2.184796854521625,
+      "grad_norm": 0.07184480875730515,
+      "learning_rate": 0.00019717396006055802,
+      "loss": 0.0731,
       "step": 2500
     },
     {
-      "epoch": 0.5461496450027308,
-      "eval_loss": 0.27210181951522827,
-      "eval_runtime": 53.5513,
-      "eval_samples_per_second": 1.737,
-      "eval_steps_per_second": 1.737,
+      "epoch": 2.184796854521625,
+      "eval_loss": 0.2528606653213501,
+      "eval_runtime": 49.8116,
+      "eval_samples_per_second": 1.867,
+      "eval_steps_per_second": 1.867,
       "step": 2500
     }
   ],
   "logging_steps": 500,
-  "max_steps": 228850,
+  "max_steps": 57200,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 50,
   "save_steps": 500,
-  "total_flos": 5.209536541001318e+16,
-  "train_batch_size": 1,
+  "total_flos": 2.1111966032161997e+17,
+  "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null
 }

checkpoint-2500/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:06fcd1ea98f2962639cdc05daeb092ebcef4b59d4ce2a280e465a4797d3e84ff
+oid sha256:623ba81014c1eb50f2c15b8946e12738563128d0fbbc243c8fbacd7dba6cd4b4
 size 5048