tokenizer_config.json

{
  "tokenizer_class": "PreTrainedTokenizerFast",
  "model_type": "unigram",
  "bos_token": "<s>",
  "eos_token": "</s>",
  "unk_token": "<unk>",
  "pad_token": "<pad>",
  "additional_special_tokens": [
    "<|system|>",
    "<|user|>",
    "<|assistant|>",
    "<|end|>",
    "<|sep|>",
    "<think>",
    "</think>",
    "<|step|>",
    "<|reflection|>",
    "<tool_call>",
    "</tool_call>",
    "<tool_response>",
    "</tool_response>",
    "<|code|>",
    "<|fim_prefix|>",
    "<|fim_middle|>",
    "<|fim_suffix|>",
    "<|tr|>",
    "<|en|>",
    "<|context|>",
    "<|/context|>",
    "<|image|>",
    "<|audio|>",
    "<|video|>",
    "<|file|>",
    "<|json|>",
    "<|table|>",
    "<|cite|>",
    "<|reserved_0|>",
    "<|reserved_1|>",
    "<|reserved_2|>",
    "<|reserved_3|>",
    "<|reserved_4|>",
    "<|reserved_5|>",
    "<|reserved_6|>",
    "<|reserved_7|>",
    "<|reserved_8|>",
    "<|reserved_9|>",
    "<|reserved_10|>",
    "<|reserved_11|>",
    "<|reserved_12|>",
    "<|reserved_13|>",
    "<|reserved_14|>",
    "<|reserved_15|>",
    "<|reserved_16|>",
    "<|reserved_17|>",
    "<|reserved_18|>",
    "<|reserved_19|>",
    "<|reserved_20|>",
    "<|reserved_21|>",
    "<|reserved_22|>",
    "<|reserved_23|>",
    "<|reserved_24|>",
    "<|reserved_25|>",
    "<|reserved_26|>",
    "<|reserved_27|>",
    "<|reserved_28|>",
    "<|reserved_29|>",
    "<|reserved_30|>",
    "<|reserved_31|>",
    "<|reserved_32|>",
    "<|reserved_33|>",
    "<|reserved_34|>",
    "<|reserved_35|>",
    "<|reserved_36|>",
    "<|reserved_37|>",
    "<|reserved_38|>",
    "<|reserved_39|>",
    "<|reserved_40|>",
    "<|reserved_41|>",
    "<|reserved_42|>",
    "<|reserved_43|>",
    "<|reserved_44|>",
    "<|reserved_45|>",
    "<|reserved_46|>",
    "<|reserved_47|>",
    "<|reserved_48|>",
    "<|reserved_49|>",
    "<|reserved_50|>",
    "<|reserved_51|>",
    "<|reserved_52|>",
    "<|reserved_53|>",
    "<|reserved_54|>",
    "<|reserved_55|>",
    "<|reserved_56|>",
    "<|reserved_57|>",
    "<|reserved_58|>",
    "<|reserved_59|>",
    "<|reserved_60|>",
    "<|reserved_61|>",
    "<|reserved_62|>",
    "<|reserved_63|>",
    "<|reserved_64|>",
    "<|reserved_65|>",
    "<|reserved_66|>",
    "<|reserved_67|>",
    "<|reserved_68|>",
    "<|reserved_69|>",
    "<|reserved_70|>",
    "<|reserved_71|>",
    "<|reserved_72|>",
    "<|reserved_73|>",
    "<|reserved_74|>",
    "<|reserved_75|>",
    "<|reserved_76|>",
    "<|reserved_77|>",
    "<|reserved_78|>",
    "<|reserved_79|>",
    "<|reserved_80|>",
    "<|reserved_81|>",
    "<|reserved_82|>",
    "<|reserved_83|>",
    "<|reserved_84|>",
    "<|reserved_85|>",
    "<|reserved_86|>",
    "<|reserved_87|>",
    "<|reserved_88|>",
    "<|reserved_89|>",
    "<|reserved_90|>",
    "<|reserved_91|>",
    "<|reserved_92|>",
    "<|reserved_93|>",
    "<|reserved_94|>",
    "<|reserved_95|>",
    "<|reserved_96|>",
    "<|reserved_97|>",
    "<|reserved_98|>",
    "<|reserved_99|>",
    "<|reserved_100|>",
    "<|reserved_101|>",
    "<|reserved_102|>",
    "<|reserved_103|>",
    "<|reserved_104|>",
    "<|reserved_105|>",
    "<|reserved_106|>",
    "<|reserved_107|>",
    "<|reserved_108|>",
    "<|reserved_109|>",
    "<|reserved_110|>",
    "<|reserved_111|>",
    "<|reserved_112|>",
    "<|reserved_113|>",
    "<|reserved_114|>",
    "<|reserved_115|>",
    "<|reserved_116|>",
    "<|reserved_117|>",
    "<|reserved_118|>",
    "<|reserved_119|>",
    "<|reserved_120|>",
    "<|reserved_121|>",
    "<|reserved_122|>",
    "<|reserved_123|>",
    "<|reserved_124|>",
    "<|reserved_125|>",
    "<|reserved_126|>",
    "<|reserved_127|>",
    "<|reserved_128|>",
    "<|reserved_129|>",
    "<|reserved_130|>",
    "<|reserved_131|>",
    "<|reserved_132|>",
    "<|reserved_133|>",
    "<|reserved_134|>",
    "<|reserved_135|>",
    "<|reserved_136|>",
    "<|reserved_137|>",
    "<|reserved_138|>",
    "<|reserved_139|>",
    "<|reserved_140|>",
    "<|reserved_141|>",
    "<|reserved_142|>",
    "<|reserved_143|>",
    "<|reserved_144|>",
    "<|reserved_145|>",
    "<|reserved_146|>",
    "<|reserved_147|>",
    "<|reserved_148|>",
    "<|reserved_149|>",
    "<|reserved_150|>",
    "<|reserved_151|>",
    "<|reserved_152|>",
    "<|reserved_153|>",
    "<|reserved_154|>",
    "<|reserved_155|>",
    "<|reserved_156|>",
    "<|reserved_157|>",
    "<|reserved_158|>",
    "<|reserved_159|>",
    "<|reserved_160|>",
    "<|reserved_161|>",
    "<|reserved_162|>",
    "<|reserved_163|>",
    "<|reserved_164|>",
    "<|reserved_165|>",
    "<|reserved_166|>",
    "<|reserved_167|>",
    "<|reserved_168|>",
    "<|reserved_169|>",
    "<|reserved_170|>",
    "<|reserved_171|>",
    "<|reserved_172|>",
    "<|reserved_173|>",
    "<|reserved_174|>",
    "<|reserved_175|>",
    "<|reserved_176|>",
    "<|reserved_177|>",
    "<|reserved_178|>",
    "<|reserved_179|>",
    "<|reserved_180|>",
    "<|reserved_181|>",
    "<|reserved_182|>",
    "<|reserved_183|>",
    "<|reserved_184|>",
    "<|reserved_185|>",
    "<|reserved_186|>",
    "<|reserved_187|>",
    "<|reserved_188|>",
    "<|reserved_189|>",
    "<|reserved_190|>",
    "<|reserved_191|>",
    "<|reserved_192|>",
    "<|reserved_193|>",
    "<|reserved_194|>",
    "<|reserved_195|>",
    "<|reserved_196|>",
    "<|reserved_197|>",
    "<|reserved_198|>",
    "<|reserved_199|>",
    "<|reserved_200|>",
    "<|reserved_201|>",
    "<|reserved_202|>",
    "<|reserved_203|>",
    "<|reserved_204|>",
    "<|reserved_205|>",
    "<|reserved_206|>",
    "<|reserved_207|>",
    "<|reserved_208|>",
    "<|reserved_209|>",
    "<|reserved_210|>",
    "<|reserved_211|>",
    "<|reserved_212|>",
    "<|reserved_213|>",
    "<|reserved_214|>",
    "<|reserved_215|>",
    "<|reserved_216|>",
    "<|reserved_217|>",
    "<|reserved_218|>",
    "<|reserved_219|>",
    "<|reserved_220|>",
    "<|reserved_221|>",
    "<|reserved_222|>",
    "<|reserved_223|>",
    "<|reserved_224|>",
    "<|reserved_225|>",
    "<|reserved_226|>",
    "<|reserved_227|>",
    "<|reserved_228|>",
    "<|reserved_229|>",
    "<|reserved_230|>",
    "<|reserved_231|>",
    "<|reserved_232|>",
    "<|reserved_233|>",
    "<|reserved_234|>",
    "<|reserved_235|>",
    "<|reserved_236|>",
    "<|reserved_237|>",
    "<|reserved_238|>",
    "<|reserved_239|>",
    "<|reserved_240|>",
    "<|reserved_241|>",
    "<|reserved_242|>",
    "<|reserved_243|>",
    "<|reserved_244|>",
    "<|reserved_245|>",
    "<|reserved_246|>",
    "<|reserved_247|>",
    "<|reserved_248|>",
    "<|reserved_249|>",
    "<|reserved_250|>",
    "<|reserved_251|>",
    "<|reserved_252|>",
    "<|reserved_253|>",
    "<|reserved_254|>",
    "<|reserved_255|>",
    "<|reserved_256|>",
    "<|reserved_257|>",
    "<|reserved_258|>",
    "<|reserved_259|>",
    "<|reserved_260|>",
    "<|reserved_261|>",
    "<|reserved_262|>",
    "<|reserved_263|>",
    "<|reserved_264|>",
    "<|reserved_265|>",
    "<|reserved_266|>",
    "<|reserved_267|>",
    "<|reserved_268|>",
    "<|reserved_269|>",
    "<|reserved_270|>",
    "<|reserved_271|>",
    "<|reserved_272|>",
    "<|reserved_273|>",
    "<|reserved_274|>",
    "<|reserved_275|>",
    "<|reserved_276|>",
    "<|reserved_277|>",
    "<|reserved_278|>",
    "<|reserved_279|>",
    "<|reserved_280|>",
    "<|reserved_281|>",
    "<|reserved_282|>",
    "<|reserved_283|>",
    "<|reserved_284|>",
    "<|reserved_285|>",
    "<|reserved_286|>",
    "<|reserved_287|>",
    "<|reserved_288|>",
    "<|reserved_289|>",
    "<|reserved_290|>",
    "<|reserved_291|>",
    "<|reserved_292|>",
    "<|reserved_293|>",
    "<|reserved_294|>",
    "<|reserved_295|>",
    "<|reserved_296|>",
    "<|reserved_297|>",
    "<|reserved_298|>",
    "<|reserved_299|>",
    "<|reserved_300|>",
    "<|reserved_301|>",
    "<|reserved_302|>",
    "<|reserved_303|>",
    "<|reserved_304|>",
    "<|reserved_305|>",
    "<|reserved_306|>",
    "<|reserved_307|>",
    "<|reserved_308|>",
    "<|reserved_309|>",
    "<|reserved_310|>",
    "<|reserved_311|>",
    "<|reserved_312|>",
    "<|reserved_313|>",
    "<|reserved_314|>",
    "<|reserved_315|>",
    "<|reserved_316|>",
    "<|reserved_317|>",
    "<|reserved_318|>",
    "<|reserved_319|>",
    "<|reserved_320|>",
    "<|reserved_321|>",
    "<|reserved_322|>",
    "<|reserved_323|>",
    "<|reserved_324|>",
    "<|reserved_325|>",
    "<|reserved_326|>",
    "<|reserved_327|>",
    "<|reserved_328|>",
    "<|reserved_329|>",
    "<|reserved_330|>",
    "<|reserved_331|>",
    "<|reserved_332|>",
    "<|reserved_333|>",
    "<|reserved_334|>",
    "<|reserved_335|>",
    "<|reserved_336|>",
    "<|reserved_337|>",
    "<|reserved_338|>",
    "<|reserved_339|>",
    "<|reserved_340|>",
    "<|reserved_341|>",
    "<|reserved_342|>",
    "<|reserved_343|>",
    "<|reserved_344|>",
    "<|reserved_345|>",
    "<|reserved_346|>",
    "<|reserved_347|>",
    "<|reserved_348|>",
    "<|reserved_349|>",
    "<|reserved_350|>",
    "<|reserved_351|>",
    "<|reserved_352|>",
    "<|reserved_353|>",
    "<|reserved_354|>",
    "<|reserved_355|>",
    "<|reserved_356|>",
    "<|reserved_357|>",
    "<|reserved_358|>",
    "<|reserved_359|>",
    "<|reserved_360|>",
    "<|reserved_361|>",
    "<|reserved_362|>",
    "<|reserved_363|>",
    "<|reserved_364|>",
    "<|reserved_365|>",
    "<|reserved_366|>",
    "<|reserved_367|>",
    "<|reserved_368|>",
    "<|reserved_369|>",
    "<|reserved_370|>",
    "<|reserved_371|>",
    "<|reserved_372|>",
    "<|reserved_373|>",
    "<|reserved_374|>",
    "<|reserved_375|>",
    "<|reserved_376|>",
    "<|reserved_377|>",
    "<|reserved_378|>",
    "<|reserved_379|>",
    "<|reserved_380|>",
    "<|reserved_381|>",
    "<|reserved_382|>",
    "<|reserved_383|>",
    "<|reserved_384|>",
    "<|reserved_385|>",
    "<|reserved_386|>",
    "<|reserved_387|>",
    "<|reserved_388|>",
    "<|reserved_389|>",
    "<|reserved_390|>",
    "<|reserved_391|>",
    "<|reserved_392|>",
    "<|reserved_393|>",
    "<|reserved_394|>",
    "<|reserved_395|>",
    "<|reserved_396|>",
    "<|reserved_397|>",
    "<|reserved_398|>",
    "<|reserved_399|>",
    "<|reserved_400|>",
    "<|reserved_401|>",
    "<|reserved_402|>",
    "<|reserved_403|>",
    "<|reserved_404|>",
    "<|reserved_405|>",
    "<|reserved_406|>",
    "<|reserved_407|>",
    "<|reserved_408|>",
    "<|reserved_409|>",
    "<|reserved_410|>",
    "<|reserved_411|>",
    "<|reserved_412|>",
    "<|reserved_413|>",
    "<|reserved_414|>",
    "<|reserved_415|>",
    "<|reserved_416|>",
    "<|reserved_417|>",
    "<|reserved_418|>",
    "<|reserved_419|>",
    "<|reserved_420|>",
    "<|reserved_421|>",
    "<|reserved_422|>",
    "<|reserved_423|>",
    "<|reserved_424|>",
    "<|reserved_425|>",
    "<|reserved_426|>",
    "<|reserved_427|>",
    "<|reserved_428|>",
    "<|reserved_429|>",
    "<|reserved_430|>",
    "<|reserved_431|>",
    "<|reserved_432|>",
    "<|reserved_433|>",
    "<|reserved_434|>",
    "<|reserved_435|>",
    "<|reserved_436|>",
    "<|reserved_437|>",
    "<|reserved_438|>",
    "<|reserved_439|>",
    "<|reserved_440|>",
    "<|reserved_441|>",
    "<|reserved_442|>",
    "<|reserved_443|>",
    "<|reserved_444|>",
    "<|reserved_445|>",
    "<|reserved_446|>",
    "<|reserved_447|>",
    "<|reserved_448|>",
    "<|reserved_449|>",
    "<|reserved_450|>",
    "<|reserved_451|>",
    "<|reserved_452|>",
    "<|reserved_453|>",
    "<|reserved_454|>",
    "<|reserved_455|>",
    "<|reserved_456|>",
    "<|reserved_457|>",
    "<|reserved_458|>",
    "<|reserved_459|>",
    "<|reserved_460|>",
    "<|reserved_461|>",
    "<|reserved_462|>",
    "<|reserved_463|>",
    "<|reserved_464|>",
    "<|reserved_465|>",
    "<|reserved_466|>",
    "<|reserved_467|>",
    "<|reserved_468|>",
    "<|reserved_469|>",
    "<|reserved_470|>",
    "<|reserved_471|>",
    "<|reserved_472|>",
    "<|reserved_473|>",
    "<|reserved_474|>",
    "<|reserved_475|>",
    "<|reserved_476|>",
    "<|reserved_477|>",
    "<|reserved_478|>",
    "<|reserved_479|>",
    "<|reserved_480|>",
    "<|reserved_481|>",
    "<|reserved_482|>",
    "<|reserved_483|>",
    "<|reserved_484|>",
    "<|reserved_485|>",
    "<|reserved_486|>",
    "<|reserved_487|>",
    "<|reserved_488|>",
    "<|reserved_489|>",
    "<|reserved_490|>",
    "<|reserved_491|>",
    "<|reserved_492|>",
    "<|reserved_493|>",
    "<|reserved_494|>",
    "<|reserved_495|>",
    "<|reserved_496|>",
    "<|reserved_497|>",
    "<|reserved_498|>",
    "<|reserved_499|>",
    "<|reserved_500|>",
    "<|reserved_501|>",
    "<|reserved_502|>",
    "<|reserved_503|>",
    "<|reserved_504|>",
    "<|reserved_505|>",
    "<|reserved_506|>",
    "<|reserved_507|>",
    "<|reserved_508|>",
    "<|reserved_509|>",
    "<|reserved_510|>",
    "<|reserved_511|>",
    ".",
    ",",
    "!",
    "?",
    ";",
    ":",
    "-",
    "(",
    ")",
    "[",
    "]",
    "{",
    "}",
    "/",
    "\\",
    "\"",
    "'",
    "...",
    "–",
    "—",
    "…",
    "_",
    "*",
    "+",
    "=",
    "<",
    ">",
    "|",
    "~",
    "^",
    "`",
    "₺",
    "$",
    "€",
    "£",
    "¥",
    "₹",
    "%",
    "‰",
    "°",
    "§",
    "¶",
    "№",
    "@",
    "#",
    "&",
    "±",
    "×",
    "÷",
    "≠",
    "≤",
    "≥",
    "≈",
    "∞",
    "√",
    "∑",
    "∫",
    "∂",
    "Δ",
    "π",
    "α",
    "β",
    "γ",
    "δ",
    "ε",
    "θ",
    "λ",
    "μ",
    "σ",
    "φ",
    "ω",
    "→",
    "←",
    "↑",
    "↓",
    "↔",
    "⇒",
    "•",
    "·",
    "★",
    "☆",
    "✓",
    "✗",
    "©",
    "®",
    "™",
    "«",
    "»",
    "“",
    "”",
    "‘",
    "’",
    "‹",
    "›",
    "„",
    "‚",
    "😀",
    "😃",
    "😄",
    "😁",
    "😆",
    "😅",
    "🤣",
    "😂",
    "🙂",
    "😊",
    "😇",
    "🥰",
    "😍",
    "🤩",
    "😘",
    "😗",
    "😚",
    "😙",
    "🥲",
    "😋",
    "😛",
    "😜",
    "🤪",
    "😝",
    "🤑",
    "🤗",
    "🤭",
    "🤫",
    "🤔",
    "🤐",
    "🤨",
    "😐",
    "😑",
    "😶",
    "😏",
    "😒",
    "🙄",
    "😬",
    "😮",
    "😲",
    "😳",
    "🥺",
    "😦",
    "😧",
    "😨",
    "😰",
    "😥",
    "😢",
    "😭",
    "😱",
    "😖",
    "😣",
    "😞",
    "😓",
    "😩",
    "😫",
    "🥱",
    "😤",
    "😡",
    "😠",
    "🤬",
    "😈",
    "👿",
    "💀",
    "💩",
    "🤡",
    "👻",
    "👽",
    "👾",
    "🤖",
    "👋",
    "🤚",
    "✋",
    "🖖",
    "👌",
    "🤌",
    "🤏",
    "✌️",
    "🤞",
    "🤟",
    "🤘",
    "🤙",
    "👈",
    "👉",
    "👆",
    "🖕",
    "👇",
    "☝️",
    "👍",
    "👎",
    "✊",
    "👊",
    "🤛",
    "🤜",
    "👏",
    "🙌",
    "🙏",
    "💪",
    "❤️",
    "🧡",
    "💛",
    "💚",
    "💙",
    "💜",
    "🖤",
    "🤍",
    "🤎",
    "💔",
    "❣️",
    "💕",
    "💞",
    "💓",
    "💗",
    "💖",
    "💘",
    "💝",
    "🔥",
    "✨",
    "🌟",
    "💫",
    "⭐",
    "🌈",
    "☀️",
    "🌙",
    "⚡",
    "❄️",
    "🌊",
    "🌸",
    "🌻",
    "🍀",
    "🎉",
    "🎊",
    "🎈",
    "🎁",
    "✅",
    "❌",
    "⚠️",
    "🚫",
    "❗",
    "❓",
    "‼️",
    "⁉️",
    "💯",
    "🔴",
    "🟠",
    "🟡",
    "🟢",
    "🔵",
    "🟣",
    "🟤",
    "⚫",
    "⚪",
    "🚀",
    "💻",
    "📱",
    "💡",
    "🔑",
    "🔒",
    "🔓",
    "📌",
    "📍",
    "📊",
    "📈",
    "📉",
    "🎯",
    "🏆",
    "🥇",
    "🥈",
    "🥉",
    "📸",
    "🎵",
    "🎶",
    "🎧",
    "📚",
    "✏️",
    "📝",
    "📋",
    "📎",
    "🔗",
    "💰",
    "💵",
    "☕",
    "🍕",
    "🍔",
    "🎂",
    "🍰",
    "🍩",
    "🍫",
    "🇹🇷",
    "🇺🇸",
    "🇬🇧",
    "🇩🇪",
    "🇫🇷",
    "🇪🇸",
    "🇮🇹",
    "🇯🇵"
  ],
  "clean_up_tokenization_spaces": false,
  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = '' %}{% endif %}{% if system_message != '' %}{{ '<|system|>\\n' + system_message + '<|end|>\\n' }}{% endif %}{% for message in loop_messages %}{% if message['role'] == 'user' %}{{ '<|user|>\\n' + message['content'] + '<|end|>\\n' }}{% elif message['role'] == 'assistant' %}{{ '<|assistant|>\\n' + message['content'] + '<|end|>\\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\\n' }}{% endif %}"
}