File size: 237 Bytes
06bf436
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
{
    "bos_token": "<s>",
    "cls_token": "<CLS|LLM-jp>",
    "eod_token": "</s>",
    "eos_token": "</s>",
    "mask_token": "<MASK|LLM-jp>",
    "pad_token": "<PAD|LLM-jp>",
    "sep_token": "<SEP|LLM-jp>",
    "unk_token": "<unk>"
}