Training in progress, epoch 1

adapter_config.json

@@ -12,7 +12,7 @@
   "lora_dropout": 0.05,
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 64,
+  "r": 32,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f84fa1a656df38bc6b00459218ed3924d897e60b893c4b5f90f06586a555c72f
-size 522284877
+oid sha256:97db5faccab278cc398b0e5dd82eadc5b1d7572c3f1cd212b94f5436fb211fe1
+size 261189453

config.json

@@ -21,6 +21,7 @@
   "hidden_size": 4544,
   "initializer_range": 0.02,
   "layer_norm_epsilon": 1e-05,
+  "max_position_embeddings": 2048,
   "model_type": "falcon",
   "multi_query": true,
   "new_decoder_architecture": false,
@@ -40,8 +41,10 @@
     "load_in_8bit": false,
     "quant_method": "bitsandbytes"
   },
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.34.0.dev0",
+  "transformers_version": "4.35.0.dev0",
   "use_cache": false,
-  "vocab_size": 65024
+  "vocab_size": 65027
 }

special_tokens_map.json

@@ -12,7 +12,8 @@
     ">>SUFFIX<<",
     ">>MIDDLE<<"
   ],
-  "bos_token": ">>ABSTRACT<<",
-  "eos_token": "<|endoftext|>",
-  "pad_token": "<|endoftext|>"
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "</s>",
+  "unk_token": "<unk>"
 }

tokenizer.json

@@ -7,8 +7,8 @@
       "id": 0,
       "content": ">>TITLE<<",
       "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
+      "lstrip": false,
+      "rstrip": false,
       "normalized": false,
       "special": true
     },
@@ -16,8 +16,8 @@
       "id": 1,
       "content": ">>ABSTRACT<<",
       "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
+      "lstrip": false,
+      "rstrip": false,
       "normalized": false,
       "special": true
     },
@@ -25,8 +25,8 @@
       "id": 2,
       "content": ">>INTRODUCTION<<",
       "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
+      "lstrip": false,
+      "rstrip": false,
       "normalized": false,
       "special": true
     },
@@ -34,8 +34,8 @@
       "id": 3,
       "content": ">>SUMMARY<<",
       "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
+      "lstrip": false,
+      "rstrip": false,
       "normalized": false,
       "special": true
     },
@@ -43,8 +43,8 @@
       "id": 4,
       "content": ">>COMMENT<<",
       "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
+      "lstrip": false,
+      "rstrip": false,
       "normalized": false,
       "special": true
     },
@@ -52,8 +52,8 @@
       "id": 5,
       "content": ">>ANSWER<<",
       "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
+      "lstrip": false,
+      "rstrip": false,
       "normalized": false,
       "special": true
     },
@@ -61,8 +61,8 @@
       "id": 6,
       "content": ">>QUESTION<<",
       "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
+      "lstrip": false,
+      "rstrip": false,
       "normalized": false,
       "special": true
     },
@@ -70,8 +70,8 @@
       "id": 7,
       "content": ">>DOMAIN<<",
       "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
+      "lstrip": false,
+      "rstrip": false,
       "normalized": false,
       "special": true
     },
@@ -79,8 +79,8 @@
       "id": 8,
       "content": ">>PREFIX<<",
       "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
+      "lstrip": false,
+      "rstrip": false,
       "normalized": false,
       "special": true
     },
@@ -88,8 +88,8 @@
       "id": 9,
       "content": ">>SUFFIX<<",
       "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
+      "lstrip": false,
+      "rstrip": false,
       "normalized": false,
       "special": true
     },
@@ -97,8 +97,8 @@
       "id": 10,
       "content": ">>MIDDLE<<",
       "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
+      "lstrip": false,
+      "rstrip": false,
       "normalized": false,
       "special": true
     },
@@ -106,6 +106,33 @@
       "id": 11,
       "content": "<|endoftext|>",
       "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 65024,
+      "content": "<s>",
+      "single_word": false,
+      "lstrip": true,
+      "rstrip": true,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 65025,
+      "content": "</s>",
+      "single_word": false,
+      "lstrip": true,
+      "rstrip": true,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 65026,
+      "content": "<unk>",
+      "single_word": false,
       "lstrip": true,
       "rstrip": true,
       "normalized": false,

tokenizer_config.json

@@ -3,94 +3,118 @@
   "added_tokens_decoder": {
     "0": {
       "content": ">>TITLE<<",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "1": {
       "content": ">>ABSTRACT<<",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "2": {
       "content": ">>INTRODUCTION<<",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "3": {
       "content": ">>SUMMARY<<",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "4": {
       "content": ">>COMMENT<<",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "5": {
       "content": ">>ANSWER<<",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "6": {
       "content": ">>QUESTION<<",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "7": {
       "content": ">>DOMAIN<<",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "8": {
       "content": ">>PREFIX<<",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "9": {
       "content": ">>SUFFIX<<",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "10": {
       "content": ">>MIDDLE<<",
-      "lstrip": true,
+      "lstrip": false,
       "normalized": false,
-      "rstrip": true,
+      "rstrip": false,
       "single_word": false,
       "special": true
     },
     "11": {
       "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65024": {
+      "content": "<s>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "65025": {
+      "content": "</s>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "65026": {
+      "content": "<unk>",
       "lstrip": true,
       "normalized": false,
       "rstrip": true,
@@ -112,7 +136,7 @@
     ">>MIDDLE<<"
   ],
   "clean_up_tokenization_spaces": true,
-  "eos_token": "<|endoftext|>",
+  "eos_token": "</s>",
   "model_input_names": [
     "input_ids",
     "attention_mask"

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f2511b568e83fc83e8b60277a37a878c353045e6d49eea7200681221cf75c0ff
-size 4475
+oid sha256:15f63458329d6dafc2836cb83c5e003c6b88114aa71b9350e54e71511bc1dc2f
+size 4539