Training in progress, step 2000, checkpoint

checkpoint-2000/adapter_config.json

@@ -23,12 +23,12 @@
   "revision": null,
   "target_modules": [
     "q_proj",
-    "v_proj",
-    "k_proj",
-    "up_proj",
     "down_proj",
+    "k_proj",
     "o_proj",
-    "gate_proj"
+    "v_proj",
+    "gate_proj",
+    "up_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

checkpoint-2000/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:317ab163a134c6ca9f4c6151a1c90ba86bf8c6942d5c1522a1b1423013133885
+oid sha256:ed8251762024fc2f4c034c5ff49e415a917b3f86bd01c31e74a9062e520885ce
 size 2460946960

checkpoint-2000/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:15c6e413cf4b3aaa7845cae55afbbad810ba2b72302934ec88da5ff29de0a543
-size 1831554874
+oid sha256:acb250a174d52f91277dcd8cac1526f32e1e90d86a019f08d96b706c4ac6039c
+size 1831543994

checkpoint-2000/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:161c72eb1214c3606f7a9489fd19746644bb4489c4774898a86bd9c647bbd3e4
+oid sha256:605f82ea41ee6354da7628867e8ae3cb96c4054bc67ebdeef3bf287c8d6c054e
 size 14308

checkpoint-2000/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:21749eee0f85f2114f8381bea10bb810acafb45830334830e80976c5d631fbbb
+oid sha256:f6511b2da4296df5b7cec03a71265778e50e897d818d104ae57200bc7de0cc40
 size 1064

checkpoint-2000/special_tokens_map.json

@@ -1,20 +1,4 @@
 {
-  "additional_special_tokens": [
-    {
-      "content": "[SQL]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "[/SQL]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    }
-  ],
   "bos_token": {
     "content": "<|endoftext|>",
     "lstrip": false,

checkpoint-2000/tokenizer.json

@@ -437,8 +437,8 @@
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
-      "special": true
+      "normalized": true,
+      "special": false
     },
     {
       "id": 50300,
@@ -446,8 +446,8 @@
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
-      "special": true
+      "normalized": true,
+      "special": false
     }
   ],
   "normalizer": {

checkpoint-2000/tokenizer_config.json

@@ -382,30 +382,30 @@
     "50299": {
       "content": "[SQL]",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
-      "special": true
+      "special": false
     },
     "50300": {
       "content": "[/SQL]",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
-      "special": true
+      "special": false
     }
   },
-  "additional_special_tokens": [
-    "[SQL]",
-    "[/SQL]"
-  ],
   "bos_token": "<|endoftext|>",
   "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = 'You are a helpful assistant.' %}{% endif %}{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in loop_messages %}{% if loop.index0 == 0 %}{{'<|im_start|>system\n' + system_message + '<|im_end|>\n'}}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|endoftext|>",
+  "max_length": 1024,
   "model_max_length": 4096,
   "pad_token": "<|endoftext|>",
+  "stride": 0,
   "tokenizer_class": "GPTNeoXTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "<|endoftext|>"
 }

checkpoint-2000/trainer_state.json

@@ -1,7 +1,7 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.4369197160021846,
+  "epoch": 1.7479248580166011,
   "eval_steps": 500,
   "global_step": 2000,
   "is_hyper_param_search": false,
@@ -9,73 +9,73 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.10922992900054615,
-      "grad_norm": 0.9561710357666016,
-      "learning_rate": 1.4564520827264783e-05,
-      "loss": 0.7634,
+      "epoch": 0.46,
+      "grad_norm": 0.36198097467422485,
+      "learning_rate": 6.0716454159077105e-05,
+      "loss": 0.4455,
       "step": 500
     },
     {
-      "epoch": 0.10922992900054615,
-      "eval_loss": 0.3866455852985382,
-      "eval_runtime": 53.8542,
-      "eval_samples_per_second": 1.727,
-      "eval_steps_per_second": 1.727,
+      "epoch": 0.46,
+      "eval_loss": 0.6183120608329773,
+      "eval_runtime": 195.3588,
+      "eval_samples_per_second": 2.37,
+      "eval_steps_per_second": 2.37,
       "step": 500
     },
     {
-      "epoch": 0.2184598580010923,
-      "grad_norm": 0.6381384134292603,
-      "learning_rate": 2.9129041654529566e-05,
-      "loss": 0.2265,
+      "epoch": 0.91,
+      "grad_norm": 0.1898297369480133,
+      "learning_rate": 0.00012143290831815421,
+      "loss": 0.0995,
       "step": 1000
     },
     {
-      "epoch": 0.2184598580010923,
-      "eval_loss": 0.28045815229415894,
-      "eval_runtime": 53.8781,
-      "eval_samples_per_second": 1.726,
-      "eval_steps_per_second": 1.726,
+      "epoch": 0.91,
+      "eval_loss": 0.6644699573516846,
+      "eval_runtime": 195.5381,
+      "eval_samples_per_second": 2.368,
+      "eval_steps_per_second": 2.368,
       "step": 1000
     },
     {
-      "epoch": 0.3276897870016384,
-      "grad_norm": 0.80877685546875,
-      "learning_rate": 4.3693562481794354e-05,
-      "loss": 0.13,
+      "epoch": 1.37,
+      "grad_norm": 0.25010770559310913,
+      "learning_rate": 0.00018214936247723133,
+      "loss": 0.0858,
       "step": 1500
     },
     {
-      "epoch": 0.3276897870016384,
-      "eval_loss": 0.24824748933315277,
-      "eval_runtime": 53.7906,
-      "eval_samples_per_second": 1.729,
-      "eval_steps_per_second": 1.729,
+      "epoch": 1.37,
+      "eval_loss": 0.6735700368881226,
+      "eval_runtime": 195.292,
+      "eval_samples_per_second": 2.371,
+      "eval_steps_per_second": 2.371,
       "step": 1500
     },
     {
-      "epoch": 0.4369197160021846,
-      "grad_norm": 0.7288264632225037,
-      "learning_rate": 5.8228954267404603e-05,
-      "loss": 0.1067,
+      "epoch": 1.7479248580166011,
+      "grad_norm": 0.12149681895971298,
+      "learning_rate": 0.00019897628145050826,
+      "loss": 0.0836,
       "step": 2000
     },
     {
-      "epoch": 0.4369197160021846,
-      "eval_loss": 0.24914385378360748,
-      "eval_runtime": 53.8469,
-      "eval_samples_per_second": 1.727,
-      "eval_steps_per_second": 1.727,
+      "epoch": 1.7479248580166011,
+      "eval_loss": 0.2921118438243866,
+      "eval_runtime": 49.9169,
+      "eval_samples_per_second": 1.863,
+      "eval_steps_per_second": 1.863,
       "step": 2000
     }
   ],
   "logging_steps": 500,
-  "max_steps": 228850,
+  "max_steps": 57200,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 50,
   "save_steps": 500,
-  "total_flos": 4.168017026968781e+16,
-  "train_batch_size": 1,
+  "total_flos": 1.6859431125624422e+17,
+  "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null
 }

checkpoint-2000/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:06fcd1ea98f2962639cdc05daeb092ebcef4b59d4ce2a280e465a4797d3e84ff
+oid sha256:623ba81014c1eb50f2c15b8946e12738563128d0fbbc243c8fbacd7dba6cd4b4
 size 5048