tokentokentokena / added_tokens.json
cakiki's picture
Upload tokenizer
0882878
{
"<commit_after>": 17,
"<commit_before>": 15,
"<commit_msg>": 16,
"<empty_output>": 14,
"<filename>": 5,
"<fim_middle>": 2,
"<fim_pad>": 4,
"<fim_prefix>": 1,
"<fim_suffix>": 3,
"<gh_stars>": 6,
"<issue_closed>": 9,
"<issue_comment>": 8,
"<issue_start>": 7,
"<jupyter_code>": 12,
"<jupyter_output>": 13,
"<jupyter_start>": 10,
"<jupyter_text>": 11,
"<reponame>": 18,
"<|endoftext|>": 0
}