Update model, now trined with OpenAssistant dataset in ChatML format

config.json

@@ -1,11 +1,10 @@
 {
-  "_name_or_path": "Locutusque/TinyMistral-248M",
+  "_name_or_path": "./TinyMistral-248M/",
   "architectures": [
     "MistralForCausalLM"
   ],
-  "bos_token_id": 32000,
-  "eos_token_id": 32001,
-  "pad_token_id": 32002,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
   "hidden_act": "silu",
   "hidden_size": 1024,
   "initializer_range": 0.02,
@@ -19,8 +18,8 @@
   "rope_theta": 10000.0,
   "sliding_window": 32,
   "tie_word_embeddings": false,
-  "torch_dtype": "float16",
+  "torch_dtype": "float32",
   "transformers_version": "4.34.1",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 32003
 }

generation_config.json

@@ -1,7 +1,6 @@
 {
   "_from_model_config": true,
-  "bos_token_id": 32000,
-  "eos_token_id": 32001,
-  "pad_token_id": 32002,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
   "transformers_version": "4.34.1"
 }

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e01fe28eb61f7d317f3236e36c88be43e136ef4ae66a424e8c912a98141aba2
+size 992115782

special_tokens_map.json

@@ -1,13 +1,13 @@
 {
   "bos_token": {
-    "content": "<|bos|>",
+    "content": "<s>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "eos_token": {
-    "content": "<|endoftext|>",
+    "content": "</s>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer.json

@@ -80,7 +80,7 @@
     "single": [
       {
         "SpecialToken": {
-          "id": "<|bos|>",
+          "id": "<s>",
           "type_id": 0
         }
       },
@@ -94,7 +94,7 @@
     "pair": [
       {
         "SpecialToken": {
-          "id": "<|bos|>",
+          "id": "<s>",
           "type_id": 0
         }
       },
@@ -106,7 +106,7 @@
       },
       {
         "SpecialToken": {
-          "id": "<|bos|>",
+          "id": "<s>",
           "type_id": 1
         }
       },
@@ -118,13 +118,13 @@
       }
     ],
     "special_tokens": {
-      "<|bos|>": {
-        "id": "<|bos|>",
+      "<s>": {
+        "id": "<s>",
         "ids": [
-          32000
+          1
         ],
         "tokens": [
-          "<|bos|>"
+          "<s>"
         ]
       }
     }

tokenizer_config.json

@@ -1,6 +1,4 @@
 {
-  "add_bos_token": true,
-  "add_eos_token": false,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
@@ -52,15 +50,19 @@
     }
   },
   "additional_special_tokens": [],
-  "bos_token": "<|bos|>",
+  "bos_token": "<s>",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "<|endoftext|>",
+  "eos_token": "</s>",
   "legacy": true,
-  "model_max_length": 1000000000000000019884624838656,
+  "max_length": 1536,
+  "model_max_length": 4096,
   "pad_token": "[PAD]",
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
+  "stride": 0,
   "tokenizer_class": "LlamaTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "<unk>",
   "use_default_system_prompt": true
 }

training_params.json

@@ -1,20 +1,20 @@
 {
-    "model": "Felladrin/TinyMistral-248M-Evol-Instruct",
-    "data_path": "data/",
-    "project_name": "TinyMistral-248M-Evol-Instruct",
+    "model": "./TinyMistral-248M/",
+    "data_path": "OpenAssistant/oasst_top1_2023-08-25",
+    "project_name": "trained-model",
     "train_split": "train",
-    "valid_split": null,
+    "valid_split": "test",
     "text_column": "text",
     "rejected_text_column": "rejected",
     "token": null,
-    "lr": 0.0002,
-    "epochs": 1,
-    "batch_size": 12,
-    "warmup_ratio": 0.1,
-    "gradient_accumulation": 4,
+    "lr": 1e-05,
+    "epochs": 5,
+    "batch_size": 2,
+    "warmup_ratio": 0.05,
+    "gradient_accumulation": 8,
     "optimizer": "adamw_torch",
-    "scheduler": "linear",
-    "weight_decay": 0.01,
+    "scheduler": "constant",
+    "weight_decay": 0.0,
     "max_grad_norm": 1.0,
     "seed": 42,
     "add_eos_token": false,
@@ -23,20 +23,20 @@
     "lora_r": 16,
     "lora_alpha": 32,
     "lora_dropout": 0.05,
-    "logging_steps": -1,
-    "evaluation_strategy": "epoch",
-    "save_total_limit": 1,
-    "save_strategy": "epoch",
+    "logging_steps": 50,
+    "evaluation_strategy": "steps",
+    "save_total_limit": 2,
+    "save_strategy": "steps",
     "auto_find_batch_size": false,
     "fp16": false,
     "push_to_hub": false,
-    "use_int8": true,
-    "model_max_length": 1024,
+    "use_int8": false,
+    "model_max_length": 4096,
     "repo_id": null,
     "use_int4": false,
     "trainer": "sft",
     "target_modules": null,
-    "merge_adapter": true,
+    "merge_adapter": false,
     "username": null,
     "use_flash_attention_2": false,
     "log": "none",