karakuri-lm-8x7b-instruct-v0.1 / special_tokens_map.json
ynakashima's picture
Upload tokenizer
ef7e830 verified
raw
history blame
3.71 kB
{
"additional_special_tokens": [
"<|START_OF_TURN_TOKEN|>",
"<|END_OF_TURN_TOKEN|>",
"<|USER_TOKEN|>",
"<|CHATBOT_TOKEN|>",
"<|SYSTEM_TOKEN|>",
"<|EXTRA_5_TOKEN|>",
"<|EXTRA_6_TOKEN|>",
"<|EXTRA_7_TOKEN|>",
"<|EXTRA_8_TOKEN|>",
"<|EXTRA_9_TOKEN|>",
"<|EXTRA_10_TOKEN|>",
"<|EXTRA_11_TOKEN|>",
"<|EXTRA_12_TOKEN|>",
"<|EXTRA_13_TOKEN|>",
"<|EXTRA_14_TOKEN|>",
"<|EXTRA_15_TOKEN|>",
"<|EXTRA_16_TOKEN|>",
"<|EXTRA_17_TOKEN|>",
"<|EXTRA_18_TOKEN|>",
"<|EXTRA_19_TOKEN|>",
"<|EXTRA_20_TOKEN|>",
"<|EXTRA_21_TOKEN|>",
"<|EXTRA_22_TOKEN|>",
"<|EXTRA_23_TOKEN|>",
"<|EXTRA_24_TOKEN|>",
"<|EXTRA_25_TOKEN|>",
"<|EXTRA_26_TOKEN|>",
"<|EXTRA_27_TOKEN|>",
"<|EXTRA_28_TOKEN|>",
"<|EXTRA_29_TOKEN|>",
"<|EXTRA_30_TOKEN|>",
"<|EXTRA_31_TOKEN|>",
"<|EXTRA_32_TOKEN|>",
"<|EXTRA_33_TOKEN|>",
"<|EXTRA_34_TOKEN|>",
"<|EXTRA_35_TOKEN|>",
"<|EXTRA_36_TOKEN|>",
"<|EXTRA_37_TOKEN|>",
"<|EXTRA_38_TOKEN|>",
"<|EXTRA_39_TOKEN|>",
"<|EXTRA_40_TOKEN|>",
"<|EXTRA_41_TOKEN|>",
"<|EXTRA_42_TOKEN|>",
"<|EXTRA_43_TOKEN|>",
"<|EXTRA_44_TOKEN|>",
"<|EXTRA_45_TOKEN|>",
"<|EXTRA_46_TOKEN|>",
"<|EXTRA_47_TOKEN|>",
"<|EXTRA_48_TOKEN|>",
"<|EXTRA_49_TOKEN|>",
"<|EXTRA_50_TOKEN|>",
"<|EXTRA_51_TOKEN|>",
"<|EXTRA_52_TOKEN|>",
"<|EXTRA_53_TOKEN|>",
"<|EXTRA_54_TOKEN|>",
"<|EXTRA_55_TOKEN|>",
"<|EXTRA_56_TOKEN|>",
"<|EXTRA_57_TOKEN|>",
"<|EXTRA_58_TOKEN|>",
"<|EXTRA_59_TOKEN|>",
"<|EXTRA_60_TOKEN|>",
"<|EXTRA_61_TOKEN|>",
"<|EXTRA_62_TOKEN|>",
"<|EXTRA_63_TOKEN|>",
"<|EXTRA_64_TOKEN|>",
"<|EXTRA_65_TOKEN|>",
"<|EXTRA_66_TOKEN|>",
"<|EXTRA_67_TOKEN|>",
"<|EXTRA_68_TOKEN|>",
"<|EXTRA_69_TOKEN|>",
"<|EXTRA_70_TOKEN|>",
"<|EXTRA_71_TOKEN|>",
"<|EXTRA_72_TOKEN|>",
"<|EXTRA_73_TOKEN|>",
"<|EXTRA_74_TOKEN|>",
"<|EXTRA_75_TOKEN|>",
"<|EXTRA_76_TOKEN|>",
"<|EXTRA_77_TOKEN|>",
"<|EXTRA_78_TOKEN|>",
"<|EXTRA_79_TOKEN|>",
"<|EXTRA_80_TOKEN|>",
"<|EXTRA_81_TOKEN|>",
"<|EXTRA_82_TOKEN|>",
"<|EXTRA_83_TOKEN|>",
"<|EXTRA_84_TOKEN|>",
"<|EXTRA_85_TOKEN|>",
"<|EXTRA_86_TOKEN|>",
"<|EXTRA_87_TOKEN|>",
"<|EXTRA_88_TOKEN|>",
"<|EXTRA_89_TOKEN|>",
"<|EXTRA_90_TOKEN|>",
"<|EXTRA_91_TOKEN|>",
"<|EXTRA_92_TOKEN|>",
"<|EXTRA_93_TOKEN|>",
"<|EXTRA_94_TOKEN|>",
"<|EXTRA_95_TOKEN|>",
"<|EXTRA_96_TOKEN|>",
"<|EXTRA_97_TOKEN|>",
"<|EXTRA_98_TOKEN|>",
"<|EXTRA_99_TOKEN|>",
"<|EXTRA_100_TOKEN|>",
"<|EXTRA_101_TOKEN|>",
"<|EXTRA_102_TOKEN|>",
"<|EXTRA_103_TOKEN|>",
"<|EXTRA_104_TOKEN|>",
"<|EXTRA_105_TOKEN|>",
"<|EXTRA_106_TOKEN|>",
"<|EXTRA_107_TOKEN|>",
"<|EXTRA_108_TOKEN|>",
"<|EXTRA_109_TOKEN|>",
"<|EXTRA_110_TOKEN|>",
"<|EXTRA_111_TOKEN|>",
"<|EXTRA_112_TOKEN|>",
"<|EXTRA_113_TOKEN|>",
"<|EXTRA_114_TOKEN|>",
"<|EXTRA_115_TOKEN|>",
"<|EXTRA_116_TOKEN|>",
"<|EXTRA_117_TOKEN|>",
"<|EXTRA_118_TOKEN|>",
"<|EXTRA_119_TOKEN|>",
"<|EXTRA_120_TOKEN|>",
"<|EXTRA_121_TOKEN|>",
"<|EXTRA_122_TOKEN|>",
"<|EXTRA_123_TOKEN|>",
"<|EXTRA_124_TOKEN|>",
"<|EXTRA_125_TOKEN|>",
"<|EXTRA_126_TOKEN|>",
"<|EXTRA_127_TOKEN|>"
],
"bos_token": {
"content": "<s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"eos_token": "<|END_OF_TURN_TOKEN|>",
"unk_token": {
"content": "<unk>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
}
}