donut-web / added_tokens.json
laverdes's picture
feat: new tokenizer and config
29b0e23
raw
history blame
1.06 kB
{
"</s_Abstract>": 57564,
"</s_Address>": 57538,
"</s_Advertisement>": 57556,
"</s_Author>": 57546,
"</s_Caption>": 57530,
"</s_Chart>": 57560,
"</s_Field-Name>": 57548,
"</s_Footer>": 57542,
"</s_Header>": 57552,
"</s_Headline>": 57536,
"</s_Link>": 57528,
"</s_List-item>": 57540,
"</s_Metadata>": 57558,
"</s_Misc>": 57534,
"</s_Page number>": 57562,
"</s_Subheadline>": 57532,
"</s_Table>": 57550,
"</s_Text>": 57526,
"</s_Title>": 57554,
"</s_Value>": 57544,
"<s_Abstract>": 57563,
"<s_Address>": 57537,
"<s_Advertisement>": 57555,
"<s_Author>": 57545,
"<s_Caption>": 57529,
"<s_Chart>": 57559,
"<s_Field-Name>": 57547,
"<s_Footer>": 57541,
"<s_Header>": 57551,
"<s_Headline>": 57535,
"<s_Link>": 57527,
"<s_List-item>": 57539,
"<s_Metadata>": 57557,
"<s_Misc>": 57533,
"<s_Page number>": 57561,
"<s_Subheadline>": 57531,
"<s_Table>": 57549,
"<s_Text>": 57525,
"<s_Title>": 57553,
"<s_Value>": 57543,
"<s_iitcdip>": 57523,
"<s_synthdog>": 57524,
"<sep/>": 57522
}