tokenizer_config.json

{
  "add_prefix_space": false,
  "audio_end_token": "<longcat_audio_end>",
  "audio_pad_token": "<longcat_audio_pad>",
  "audio_start_token": "<longcat_audio_start>",
  "backend": "tokenizers",
  "bos_token": "<longcat_s>",
  "clean_up_tokenization_spaces": false,
  "eos_token": "</longcat_s>",
  "extra_special_tokens": [
    "<mask_131048>",
    "<mask_131049>",
    "<mask_131050>",
    "<mask_131051>",
    "<mask_131052>",
    "<mask_131053>",
    "<mask_131054>",
    "<mask_131055>",
    "<mask_131056>",
    "<mask_131057>",
    "<mask_131058>",
    "<mask_131059>",
    "<mask_131060>",
    "<mask_131061>",
    "<mask_131062>",
    "<mask_131063>",
    "<mask_131064>",
    "<mask_131065>",
    "<longcat_img_token_size>",
    "</longcat_img_token_size>",
    "<mask_131068>",
    "<mask_131069>",
    "<mask_131070>",
    "<mask_131071>",
    "<longcat_point_start>",
    "<longcat_point_end>",
    "<longcat_point_delim>",
    "<longcat_polygon_start>",
    "<longcat_polygon_end>",
    "<mask_131077>",
    "<mask_131078>",
    "<longcat_audio_start>",
    "<longcat_audio_end>",
    "<longcat_audio_pad>",
    "<longcat_img_start>",
    "<longcat_img_end>",
    "<longcat_img_pad>",
    "<longcat_img_newline>",
    "<longcat_box_start>",
    "<longcat_box_end>",
    "<longcat_box_delim>",
    "<longcat_ref_start>",
    "<longcat_ref_end>",
    "<longcat_img_delim>",
    "<longcat_audio_delim>",
    "<longcat_video_palce>",
    "<longcat_video_start>",
    "<longcat_video_end>",
    "<longcat_audiotext_start>",
    "<longcat_audiotext_end>",
    "<longcat_audiotext_pad>",
    "<longcat_audiogen_start>",
    "<longcat_audiogen_end>"
  ],
  "image_end_token": "<longcat_img_end>",
  "image_newline_token": "<longcat_img_newline>",
  "image_pad_token": "<longcat_img_pad>",
  "image_start_token": "<longcat_img_start>",
  "is_local": true,
  "model_max_length": 131072,
  "model_specific_special_tokens": {
    "audio_end_token": "<longcat_audio_end>",
    "audio_pad_token": "<longcat_audio_pad>",
    "audio_start_token": "<longcat_audio_start>",
    "image_end_token": "<longcat_img_end>",
    "image_newline_token": "<longcat_img_newline>",
    "image_pad_token": "<longcat_img_pad>",
    "image_start_token": "<longcat_img_start>"
  },
  "pad_token": "<longcat_pad>",
  "sp_model_kwargs": {},
  "tokenizer_class": "TokenizersBackend",
  "tool_parser_type": "longcat",
  "unk_token": "<longcat_unk>"
}