add tokenizer to v2.1

README.md

@@ -1,17 +1,17 @@
 ---
-library_name: transformers
-tags:
-- code
-- hpc
-- parallel
-- axonn
 datasets:
 - hpcgroup/hpc-instruct
 - ise-uiuc/Magicoder-OSS-Instruct-75K
 - nickrosh/Evol-Instruct-Code-80k-v1
 language:
 - en
+library_name: transformers
 pipeline_tag: text-generation
+tags:
+- code
+- hpc
+- parallel
+- axonn
 ---
 
 # HPC-Coder-v2

tokenizer.json

@@ -321,6 +321,7 @@
     "end_of_word_suffix": null,
     "fuse_unk": false,
     "byte_fallback": false,
+    "ignore_merges": false,
     "vocab": {
       "!": 0,
       "\"": 1,

tokenizer_config.json

@@ -1,6 +1,7 @@
 {
   "add_bos_token": true,
   "add_eos_token": false,
+  "add_prefix_space": null,
   "added_tokens_decoder": {
     "32000": {
       "content": "õ",