tini_llama_full / added_tokens.json
AlexWortega's picture
Upload tokenizer
9d23390 verified
raw
history blame
1.06 kB
{
"<|future_token_32|>": 60223,
"<|future_token_33|>": 60224,
"<|future_token_34|>": 60225,
"<|future_token_35|>": 60226,
"<|future_token_36|>": 60227,
"<|future_token_37|>": 60228,
"<|future_token_38|>": 60229,
"<|future_token_39|>": 60230,
"<|future_token_40|>": 60231,
"<|future_token_41|>": 60232,
"<|future_token_42|>": 60233,
"<|future_token_43|>": 60234,
"<|future_token_44|>": 60235,
"<|future_token_45|>": 60236,
"<|future_token_46|>": 60237,
"<|future_token_47|>": 60238,
"<|future_token_48|>": 60239,
"<|future_token_49|>": 60240,
"<|future_token_50|>": 60241,
"<|future_token_51|>": 60242,
"<|future_token_52|>": 60243,
"<|future_token_53|>": 60244,
"<|future_token_54|>": 60245,
"<|future_token_55|>": 60246,
"<|future_token_56|>": 60247,
"<|future_token_57|>": 60248,
"<|future_token_58|>": 60249,
"<|future_token_59|>": 60250,
"<|future_token_60|>": 60251,
"<|future_token_61|>": 60252,
"<|future_token_62|>": 60253,
"<|future_token_63|>": 60254,
"<|future_token_64|>": 60255
}