Upload tokenizer

tokenizer.json

@@ -9,7 +9,7 @@
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
+      "normalized": true,
       "special": true
     },
     {
@@ -18,7 +18,7 @@
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
+      "normalized": true,
       "special": true
     },
     {
@@ -27,7 +27,7 @@
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
+      "normalized": true,
       "special": true
     }
   ],

tokenizer_config.json

@@ -1,6 +1,5 @@
 {
-  "add_bos_token": true,
-  "add_eos_token": false,
+  "additional_special_tokens": [],
   "bos_token": {
     "__type": "AddedToken",
     "content": "<s>",
@@ -18,10 +17,16 @@
     "rstrip": false,
     "single_word": false
   },
+  "eot_token": null,
+  "fill_token": "<FILL_ME>",
   "legacy": null,
+  "middle_token": null,
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": null,
+  "prefix_token": null,
   "sp_model_kwargs": {},
+  "suffix_first": false,
+  "suffix_token": null,
   "tokenizer_class": "CodeLlamaTokenizer",
   "unk_token": {
     "__type": "AddedToken",
@@ -30,5 +35,6 @@
     "normalized": true,
     "rstrip": false,
     "single_word": false
-  }
+  },
+  "use_default_system_prompt": false
 }