Add chat_template

tokenizer_config.json

@@ -2049,13 +2049,273 @@
       "special": true
     }
   },
+  "additional_special_tokens": [
+    "<|begin_of_text|>",
+    "<|end_of_text|>",
+    "<|reserved_special_token_0|>",
+    "<|reserved_special_token_1|>",
+    "<|reserved_special_token_2|>",
+    "<|reserved_special_token_3|>",
+    "<|start_header_id|>",
+    "<|end_header_id|>",
+    "<|reserved_special_token_4|>",
+    "<|eot_id|>",
+    "<|reserved_special_token_5|>",
+    "<|reserved_special_token_6|>",
+    "<|reserved_special_token_7|>",
+    "<|reserved_special_token_8|>",
+    "<|reserved_special_token_9|>",
+    "<|reserved_special_token_10|>",
+    "<|reserved_special_token_11|>",
+    "<|reserved_special_token_12|>",
+    "<|reserved_special_token_13|>",
+    "<|reserved_special_token_14|>",
+    "<|reserved_special_token_15|>",
+    "<|reserved_special_token_16|>",
+    "<|reserved_special_token_17|>",
+    "<|reserved_special_token_18|>",
+    "<|reserved_special_token_19|>",
+    "<|reserved_special_token_20|>",
+    "<|reserved_special_token_21|>",
+    "<|reserved_special_token_22|>",
+    "<|reserved_special_token_23|>",
+    "<|reserved_special_token_24|>",
+    "<|reserved_special_token_25|>",
+    "<|reserved_special_token_26|>",
+    "<|reserved_special_token_27|>",
+    "<|reserved_special_token_28|>",
+    "<|reserved_special_token_29|>",
+    "<|reserved_special_token_30|>",
+    "<|reserved_special_token_31|>",
+    "<|reserved_special_token_32|>",
+    "<|reserved_special_token_33|>",
+    "<|reserved_special_token_34|>",
+    "<|reserved_special_token_35|>",
+    "<|reserved_special_token_36|>",
+    "<|reserved_special_token_37|>",
+    "<|reserved_special_token_38|>",
+    "<|reserved_special_token_39|>",
+    "<|reserved_special_token_40|>",
+    "<|reserved_special_token_41|>",
+    "<|reserved_special_token_42|>",
+    "<|reserved_special_token_43|>",
+    "<|reserved_special_token_44|>",
+    "<|reserved_special_token_45|>",
+    "<|reserved_special_token_46|>",
+    "<|reserved_special_token_47|>",
+    "<|reserved_special_token_48|>",
+    "<|reserved_special_token_49|>",
+    "<|reserved_special_token_50|>",
+    "<|reserved_special_token_51|>",
+    "<|reserved_special_token_52|>",
+    "<|reserved_special_token_53|>",
+    "<|reserved_special_token_54|>",
+    "<|reserved_special_token_55|>",
+    "<|reserved_special_token_56|>",
+    "<|reserved_special_token_57|>",
+    "<|reserved_special_token_58|>",
+    "<|reserved_special_token_59|>",
+    "<|reserved_special_token_60|>",
+    "<|reserved_special_token_61|>",
+    "<|reserved_special_token_62|>",
+    "<|reserved_special_token_63|>",
+    "<|reserved_special_token_64|>",
+    "<|reserved_special_token_65|>",
+    "<|reserved_special_token_66|>",
+    "<|reserved_special_token_67|>",
+    "<|reserved_special_token_68|>",
+    "<|reserved_special_token_69|>",
+    "<|reserved_special_token_70|>",
+    "<|reserved_special_token_71|>",
+    "<|reserved_special_token_72|>",
+    "<|reserved_special_token_73|>",
+    "<|reserved_special_token_74|>",
+    "<|reserved_special_token_75|>",
+    "<|reserved_special_token_76|>",
+    "<|reserved_special_token_77|>",
+    "<|reserved_special_token_78|>",
+    "<|reserved_special_token_79|>",
+    "<|reserved_special_token_80|>",
+    "<|reserved_special_token_81|>",
+    "<|reserved_special_token_82|>",
+    "<|reserved_special_token_83|>",
+    "<|reserved_special_token_84|>",
+    "<|reserved_special_token_85|>",
+    "<|reserved_special_token_86|>",
+    "<|reserved_special_token_87|>",
+    "<|reserved_special_token_88|>",
+    "<|reserved_special_token_89|>",
+    "<|reserved_special_token_90|>",
+    "<|reserved_special_token_91|>",
+    "<|reserved_special_token_92|>",
+    "<|reserved_special_token_93|>",
+    "<|reserved_special_token_94|>",
+    "<|reserved_special_token_95|>",
+    "<|reserved_special_token_96|>",
+    "<|reserved_special_token_97|>",
+    "<|reserved_special_token_98|>",
+    "<|reserved_special_token_99|>",
+    "<|reserved_special_token_100|>",
+    "<|reserved_special_token_101|>",
+    "<|reserved_special_token_102|>",
+    "<|reserved_special_token_103|>",
+    "<|reserved_special_token_104|>",
+    "<|reserved_special_token_105|>",
+    "<|reserved_special_token_106|>",
+    "<|reserved_special_token_107|>",
+    "<|reserved_special_token_108|>",
+    "<|reserved_special_token_109|>",
+    "<|reserved_special_token_110|>",
+    "<|reserved_special_token_111|>",
+    "<|reserved_special_token_112|>",
+    "<|reserved_special_token_113|>",
+    "<|reserved_special_token_114|>",
+    "<|reserved_special_token_115|>",
+    "<|reserved_special_token_116|>",
+    "<|reserved_special_token_117|>",
+    "<|reserved_special_token_118|>",
+    "<|reserved_special_token_119|>",
+    "<|reserved_special_token_120|>",
+    "<|reserved_special_token_121|>",
+    "<|reserved_special_token_122|>",
+    "<|reserved_special_token_123|>",
+    "<|reserved_special_token_124|>",
+    "<|reserved_special_token_125|>",
+    "<|reserved_special_token_126|>",
+    "<|reserved_special_token_127|>",
+    "<|reserved_special_token_128|>",
+    "<|reserved_special_token_129|>",
+    "<|reserved_special_token_130|>",
+    "<|reserved_special_token_131|>",
+    "<|reserved_special_token_132|>",
+    "<|reserved_special_token_133|>",
+    "<|reserved_special_token_134|>",
+    "<|reserved_special_token_135|>",
+    "<|reserved_special_token_136|>",
+    "<|reserved_special_token_137|>",
+    "<|reserved_special_token_138|>",
+    "<|reserved_special_token_139|>",
+    "<|reserved_special_token_140|>",
+    "<|reserved_special_token_141|>",
+    "<|reserved_special_token_142|>",
+    "<|reserved_special_token_143|>",
+    "<|reserved_special_token_144|>",
+    "<|reserved_special_token_145|>",
+    "<|reserved_special_token_146|>",
+    "<|reserved_special_token_147|>",
+    "<|reserved_special_token_148|>",
+    "<|reserved_special_token_149|>",
+    "<|reserved_special_token_150|>",
+    "<|reserved_special_token_151|>",
+    "<|reserved_special_token_152|>",
+    "<|reserved_special_token_153|>",
+    "<|reserved_special_token_154|>",
+    "<|reserved_special_token_155|>",
+    "<|reserved_special_token_156|>",
+    "<|reserved_special_token_157|>",
+    "<|reserved_special_token_158|>",
+    "<|reserved_special_token_159|>",
+    "<|reserved_special_token_160|>",
+    "<|reserved_special_token_161|>",
+    "<|reserved_special_token_162|>",
+    "<|reserved_special_token_163|>",
+    "<|reserved_special_token_164|>",
+    "<|reserved_special_token_165|>",
+    "<|reserved_special_token_166|>",
+    "<|reserved_special_token_167|>",
+    "<|reserved_special_token_168|>",
+    "<|reserved_special_token_169|>",
+    "<|reserved_special_token_170|>",
+    "<|reserved_special_token_171|>",
+    "<|reserved_special_token_172|>",
+    "<|reserved_special_token_173|>",
+    "<|reserved_special_token_174|>",
+    "<|reserved_special_token_175|>",
+    "<|reserved_special_token_176|>",
+    "<|reserved_special_token_177|>",
+    "<|reserved_special_token_178|>",
+    "<|reserved_special_token_179|>",
+    "<|reserved_special_token_180|>",
+    "<|reserved_special_token_181|>",
+    "<|reserved_special_token_182|>",
+    "<|reserved_special_token_183|>",
+    "<|reserved_special_token_184|>",
+    "<|reserved_special_token_185|>",
+    "<|reserved_special_token_186|>",
+    "<|reserved_special_token_187|>",
+    "<|reserved_special_token_188|>",
+    "<|reserved_special_token_189|>",
+    "<|reserved_special_token_190|>",
+    "<|reserved_special_token_191|>",
+    "<|reserved_special_token_192|>",
+    "<|reserved_special_token_193|>",
+    "<|reserved_special_token_194|>",
+    "<|reserved_special_token_195|>",
+    "<|reserved_special_token_196|>",
+    "<|reserved_special_token_197|>",
+    "<|reserved_special_token_198|>",
+    "<|reserved_special_token_199|>",
+    "<|reserved_special_token_200|>",
+    "<|reserved_special_token_201|>",
+    "<|reserved_special_token_202|>",
+    "<|reserved_special_token_203|>",
+    "<|reserved_special_token_204|>",
+    "<|reserved_special_token_205|>",
+    "<|reserved_special_token_206|>",
+    "<|reserved_special_token_207|>",
+    "<|reserved_special_token_208|>",
+    "<|reserved_special_token_209|>",
+    "<|reserved_special_token_210|>",
+    "<|reserved_special_token_211|>",
+    "<|reserved_special_token_212|>",
+    "<|reserved_special_token_213|>",
+    "<|reserved_special_token_214|>",
+    "<|reserved_special_token_215|>",
+    "<|reserved_special_token_216|>",
+    "<|reserved_special_token_217|>",
+    "<|reserved_special_token_218|>",
+    "<|reserved_special_token_219|>",
+    "<|reserved_special_token_220|>",
+    "<|reserved_special_token_221|>",
+    "<|reserved_special_token_222|>",
+    "<|reserved_special_token_223|>",
+    "<|reserved_special_token_224|>",
+    "<|reserved_special_token_225|>",
+    "<|reserved_special_token_226|>",
+    "<|reserved_special_token_227|>",
+    "<|reserved_special_token_228|>",
+    "<|reserved_special_token_229|>",
+    "<|reserved_special_token_230|>",
+    "<|reserved_special_token_231|>",
+    "<|reserved_special_token_232|>",
+    "<|reserved_special_token_233|>",
+    "<|reserved_special_token_234|>",
+    "<|reserved_special_token_235|>",
+    "<|reserved_special_token_236|>",
+    "<|reserved_special_token_237|>",
+    "<|reserved_special_token_238|>",
+    "<|reserved_special_token_239|>",
+    "<|reserved_special_token_240|>",
+    "<|reserved_special_token_241|>",
+    "<|reserved_special_token_242|>",
+    "<|reserved_special_token_243|>",
+    "<|reserved_special_token_244|>",
+    "<|reserved_special_token_245|>",
+    "<|reserved_special_token_246|>",
+    "<|reserved_special_token_247|>",
+    "<|reserved_special_token_248|>",
+    "<|reserved_special_token_249|>",
+    "<|reserved_special_token_250|>"
+  ],
   "bos_token": "<|begin_of_text|>",
+  "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}",
   "clean_up_tokenization_spaces": true,
-  "eos_token": "<|end_of_text|>",
+  "eos_token": "<|eot_id|>",
   "model_input_names": [
     "input_ids",
     "attention_mask"
   ],
   "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|end_of_text|>",
   "tokenizer_class": "PreTrainedTokenizerFast"
 }