File size: 314 Bytes
c87c295 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
B_CODE = "[CODE_START_TOK]"
E_CODE = "[/CODE_END_TOK]"
B_RESULT = "[RESULT_TOK]"
E_RESULT = "[/RESULT_TOK]"
B_INST, E_INST = "[INST]", "[/INST]"
B_SYS, E_SYS = "<<SYS>>", "<</SYS>>"
IGNORE_INDEX = -100
DEFAULT_PAD_TOKEN = "[PAD]"
DEFAULT_EOS_TOKEN = "</s>"
DEFAULT_BOS_TOKEN = "<s>"
DEFAULT_UNK_TOKEN = "<unk>"
|