Upload folder using huggingface_hub

README.md

@@ -1,6 +1,6 @@
 ---
 library_name: peft
-base_model: EleutherAI/pythia-1b
+base_model: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
 ---
 
 # Model Card for Model ID

adapter_config.json

@@ -1,7 +1,7 @@
 {
   "alpha_pattern": {},
   "auto_mapping": null,
-  "base_model_name_or_path": "EleutherAI/pythia-1b",
+  "base_model_name_or_path": "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T",
   "bias": "none",
   "fan_in_fan_out": false,
   "inference_mode": true,
@@ -19,10 +19,9 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "dense_4h_to_h",
-    "dense_h_to_4h",
-    "query_key_value",
-    "dense"
+    "gate_proj",
+    "down_proj",
+    "up_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:01f246a04293608017a7ce637adc72e865ce93286d13286382ff9af6a81f2841
-size 33572288
+oid sha256:5cf4126a093cc2744b811791cb49b4bb554a88d7e17d7a455c4e4f43f620892b
+size 32457544

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ccf51e786cdb56065c858b7f0f7a7efd17c6b4a2cd490f129f0790a702e4a60f
-size 67185658
+oid sha256:077681757457c2caab161f15957793e472c6d16b2cbfd25132fe62544cd967a8
+size 64958970

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:24ef0df5b83694239f6a83e58e420db881fdd0e0442c1d73b2233f6b05ee3044
+oid sha256:c9055ac9db4835b33df3880467ea6560e442fcdfc1be9cf492c952ca01ce18a9
 size 14244

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4372b8bc129cb832d61f89de539110be4b375eeb5fe693144c4dd880623a9aaf
+oid sha256:3bf1e902c3114bb5312c2c178dc4d942be3a06cb5931cabbb4ddb7748d0f53ee
 size 1064

special_tokens_map.json

@@ -1,21 +1,21 @@
 {
   "bos_token": {
-    "content": "<|endoftext|>",
+    "content": "<s>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "eos_token": {
-    "content": "<|endoftext|>",
+    "content": "</s>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": "<|endoftext|>",
+  "pad_token": "</s>",
   "unk_token": {
-    "content": "<|endoftext|>",
+    "content": "<unk>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -1,8 +1,9 @@
 {
-  "add_prefix_space": false,
+  "add_bos_token": true,
+  "add_eos_token": false,
   "added_tokens_decoder": {
     "0": {
-      "content": "<|endoftext|>",
+      "content": "<unk>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -10,203 +11,31 @@
       "special": true
     },
     "1": {
-      "content": "<|padding|>",
+      "content": "<s>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "50254": {
-      "content": "                        ",
+    "2": {
+      "content": "</s>",
       "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50255": {
-      "content": "                       ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50256": {
-      "content": "                      ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50257": {
-      "content": "                     ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50258": {
-      "content": "                    ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50259": {
-      "content": "                   ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50260": {
-      "content": "                  ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50261": {
-      "content": "                 ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50262": {
-      "content": "                ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50263": {
-      "content": "               ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50264": {
-      "content": "              ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50265": {
-      "content": "             ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50266": {
-      "content": "            ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50267": {
-      "content": "           ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50268": {
-      "content": "          ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50269": {
-      "content": "         ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50270": {
-      "content": "        ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50271": {
-      "content": "       ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50272": {
-      "content": "      ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50273": {
-      "content": "     ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50274": {
-      "content": "    ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50275": {
-      "content": "   ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50276": {
-      "content": "  ",
-      "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     }
   },
-  "bos_token": "<|endoftext|>",
-  "clean_up_tokenization_spaces": true,
-  "eos_token": "<|endoftext|>",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "<|endoftext|>",
-  "tokenizer_class": "GPTNeoXTokenizer",
-  "unk_token": "<|endoftext|>"
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
 }

trainer_state.json

@@ -1,164 +1,218 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9846153846153847,
+  "epoch": 0.9990680335507922,
   "eval_steps": 500,
-  "global_step": 48,
+  "global_step": 67,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.04,
+      "epoch": 0.03,
       "learning_rate": 2e-05,
-      "loss": 2.3303,
+      "loss": 2.1412,
       "step": 2
     },
     {
-      "epoch": 0.08,
+      "epoch": 0.06,
       "learning_rate": 2e-05,
-      "loss": 2.5495,
+      "loss": 1.606,
       "step": 4
     },
     {
-      "epoch": 0.12,
+      "epoch": 0.09,
       "learning_rate": 2e-05,
-      "loss": 2.5854,
+      "loss": 1.3632,
       "step": 6
     },
     {
-      "epoch": 0.16,
+      "epoch": 0.12,
       "learning_rate": 2e-05,
-      "loss": 2.612,
+      "loss": 1.3368,
       "step": 8
     },
     {
-      "epoch": 0.21,
+      "epoch": 0.15,
       "learning_rate": 2e-05,
-      "loss": 2.9175,
+      "loss": 1.4371,
       "step": 10
     },
     {
-      "epoch": 0.25,
+      "epoch": 0.18,
       "learning_rate": 2e-05,
-      "loss": 3.2592,
+      "loss": 1.4571,
       "step": 12
     },
     {
-      "epoch": 0.29,
+      "epoch": 0.21,
       "learning_rate": 2e-05,
-      "loss": 2.2769,
+      "loss": 1.9018,
       "step": 14
     },
     {
-      "epoch": 0.33,
+      "epoch": 0.24,
       "learning_rate": 2e-05,
-      "loss": 2.2051,
+      "loss": 2.2,
       "step": 16
     },
     {
-      "epoch": 0.37,
+      "epoch": 0.27,
       "learning_rate": 2e-05,
-      "loss": 2.5131,
+      "loss": 2.1347,
       "step": 18
     },
     {
-      "epoch": 0.41,
+      "epoch": 0.3,
       "learning_rate": 2e-05,
-      "loss": 2.4979,
+      "loss": 1.4048,
       "step": 20
     },
     {
-      "epoch": 0.45,
+      "epoch": 0.33,
       "learning_rate": 2e-05,
-      "loss": 2.7152,
+      "loss": 1.4807,
       "step": 22
     },
     {
-      "epoch": 0.49,
+      "epoch": 0.36,
       "learning_rate": 2e-05,
-      "loss": 2.9744,
+      "loss": 1.4868,
       "step": 24
     },
     {
-      "epoch": 0.53,
+      "epoch": 0.39,
       "learning_rate": 2e-05,
-      "loss": 2.166,
+      "loss": 1.2677,
       "step": 26
     },
     {
-      "epoch": 0.57,
+      "epoch": 0.42,
       "learning_rate": 2e-05,
-      "loss": 2.4005,
+      "loss": 1.6585,
       "step": 28
     },
     {
-      "epoch": 0.62,
+      "epoch": 0.45,
       "learning_rate": 2e-05,
-      "loss": 2.651,
+      "loss": 1.8421,
       "step": 30
     },
     {
-      "epoch": 0.66,
+      "epoch": 0.48,
       "learning_rate": 2e-05,
-      "loss": 2.3863,
+      "loss": 1.9776,
       "step": 32
     },
     {
-      "epoch": 0.7,
+      "epoch": 0.51,
       "learning_rate": 2e-05,
-      "loss": 2.4171,
+      "loss": 1.7642,
       "step": 34
     },
     {
-      "epoch": 0.74,
+      "epoch": 0.54,
       "learning_rate": 2e-05,
-      "loss": 2.7263,
+      "loss": 1.72,
       "step": 36
     },
     {
-      "epoch": 0.78,
+      "epoch": 0.57,
       "learning_rate": 2e-05,
-      "loss": 2.2354,
+      "loss": 1.3232,
       "step": 38
     },
     {
-      "epoch": 0.82,
+      "epoch": 0.6,
       "learning_rate": 2e-05,
-      "loss": 2.4808,
+      "loss": 1.4866,
       "step": 40
     },
     {
-      "epoch": 0.86,
+      "epoch": 0.63,
       "learning_rate": 2e-05,
-      "loss": 2.5683,
+      "loss": 1.2556,
       "step": 42
     },
     {
-      "epoch": 0.9,
+      "epoch": 0.66,
       "learning_rate": 2e-05,
-      "loss": 2.5998,
+      "loss": 1.3743,
       "step": 44
     },
     {
-      "epoch": 0.94,
+      "epoch": 0.69,
       "learning_rate": 2e-05,
-      "loss": 2.3712,
+      "loss": 1.7033,
       "step": 46
     },
     {
-      "epoch": 0.98,
+      "epoch": 0.72,
       "learning_rate": 2e-05,
-      "loss": 2.6373,
+      "loss": 1.8748,
       "step": 48
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2e-05,
+      "loss": 2.1102,
+      "step": 50
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2e-05,
+      "loss": 1.4274,
+      "step": 52
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 2e-05,
+      "loss": 1.317,
+      "step": 54
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 2e-05,
+      "loss": 1.2868,
+      "step": 56
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2e-05,
+      "loss": 1.4609,
+      "step": 58
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 2e-05,
+      "loss": 1.4965,
+      "step": 60
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 2e-05,
+      "loss": 1.6175,
+      "step": 62
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2e-05,
+      "loss": 1.6328,
+      "step": 64
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2e-05,
+      "loss": 1.4841,
+      "step": 66
     }
   ],
   "logging_steps": 2,
-  "max_steps": 48,
+  "max_steps": 67,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 500,
-  "total_flos": 2029337749118976.0,
+  "total_flos": 3677439948509184.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:37cd9403e23e3499bfa37285945b68a8b825a9d9793d8f63ae57f0317364c5f2
+oid sha256:52fb4bad19dcae53734968d3ad050ff304f1677c89d912c30a8116ba2a464ba1
 size 4728