tokenizer_config.json

{
  "absence_token": "<|absence|>",
  "backend": "tokenizers",
  "caption_token": "<|CAPTION|>",
  "clean_up_tokenization_spaces": true,
  "coord_token": "<|coord|>",
  "det_token": "<|DET|>",
  "detailed_caption_token": "<|DETAILED_CAPTION|>",
  "diffusion_query_token": "<|diffusion_query|>",
  "end_of_diffusion_query_token": "<|end_of_diffusion_query|>",
  "end_of_image_token": "<|end_of_image|>",
  "end_of_query_token": "<|end_of_query|>",
  "end_of_turn_token": "<|end_of_turn|>",
  "end_of_video_token": "<|end_of_video|>",
  "eos_token": "<|end_of_text|>",
  "frame_sep_token": "<|frame_sep|>",
  "image_cls_token": "<|image_cls|>",
  "image_mask_token": "<|image_mask_token|>",
  "image_reg_1_token": "<|image_reg_1|>",
  "image_reg_2_token": "<|image_reg_2|>",
  "image_reg_3_token": "<|image_reg_3|>",
  "image_reg_4_token": "<|image_reg_4|>",
  "image_reg_5_token": "<|image_reg_5|>",
  "image_reg_6_token": "<|image_reg_6|>",
  "image_reg_7_token": "<|image_reg_7|>",
  "image_reg_8_token": "<|image_reg_8|>",
  "image_row_sep_token": "<|image_row_sep|>",
  "image_token": "<|image|>",
  "is_local": true,
  "model_input_names": [
    "input_ids",
    "attention_mask"
  ],
  "model_max_length": 1000000000000000019884624838656,
  "model_specific_special_tokens": {
    "absence_token": "<|absence|>",
    "caption_token": "<|CAPTION|>",
    "coord_token": "<|coord|>",
    "det_token": "<|DET|>",
    "detailed_caption_token": "<|DETAILED_CAPTION|>",
    "diffusion_query_token": "<|diffusion_query|>",
    "end_of_diffusion_query_token": "<|end_of_diffusion_query|>",
    "end_of_image_token": "<|end_of_image|>",
    "end_of_query_token": "<|end_of_query|>",
    "end_of_turn_token": "<|end_of_turn|>",
    "end_of_video_token": "<|end_of_video|>",
    "frame_sep_token": "<|frame_sep|>",
    "image_cls_token": "<|image_cls|>",
    "image_mask_token": "<|image_mask_token|>",
    "image_reg_1_token": "<|image_reg_1|>",
    "image_reg_2_token": "<|image_reg_2|>",
    "image_reg_3_token": "<|image_reg_3|>",
    "image_reg_4_token": "<|image_reg_4|>",
    "image_reg_5_token": "<|image_reg_5|>",
    "image_reg_6_token": "<|image_reg_6|>",
    "image_reg_7_token": "<|image_reg_7|>",
    "image_reg_8_token": "<|image_reg_8|>",
    "image_row_sep_token": "<|image_row_sep|>",
    "image_token": "<|image|>",
    "object_token": "<|object|>",
    "ocr_doc_parser_token": "<|OCR_DOC_PARSER|>",
    "ocr_grounding_token": "<|OCR_GROUNDING|>",
    "ocr_plain_token": "<|OCR_PLAIN|>",
    "pad_token": "<|pad|>",
    "perceive_token": "<|perceive|>",
    "point_ref_seg_token": "<|POINT_REF_SEG|>",
    "pointing_token": "<|POINTING|>",
    "presence_token": "<|presence|>",
    "ref_seg_token": "<|REF_SEG|>",
    "seg_token": "<|seg|>",
    "semantic_seg_task_token": "<|SEMANTIC_SEG_TASK|>",
    "semantic_seg_token": "<|semantic_seg|>",
    "size_token": "<|size|>",
    "start_of_diffusion_query_token": "<|start_of_diffusion_query|>",
    "start_of_image_token": "<|start_of_image|>",
    "start_of_query_token": "<|start_of_query|>",
    "start_of_turn_token": "<|start_of_turn|>",
    "start_of_video_token": "<|start_of_video|>",
    "task_sep_token": "<|task_sep|>"
  },
  "object_token": "<|object|>",
  "ocr_doc_parser_token": "<|OCR_DOC_PARSER|>",
  "ocr_grounding_token": "<|OCR_GROUNDING|>",
  "ocr_plain_token": "<|OCR_PLAIN|>",
  "pad_token": "<|pad|>",
  "perceive_token": "<|perceive|>",
  "point_ref_seg_token": "<|POINT_REF_SEG|>",
  "pointing_token": "<|POINTING|>",
  "presence_token": "<|presence|>",
  "ref_seg_token": "<|REF_SEG|>",
  "seg_token": "<|seg|>",
  "semantic_seg_task_token": "<|SEMANTIC_SEG_TASK|>",
  "semantic_seg_token": "<|semantic_seg|>",
  "size_token": "<|size|>",
  "start_of_diffusion_query_token": "<|start_of_diffusion_query|>",
  "start_of_image_token": "<|start_of_image|>",
  "start_of_query_token": "<|start_of_query|>",
  "start_of_turn_token": "<|start_of_turn|>",
  "start_of_video_token": "<|start_of_video|>",
  "task_sep_token": "<|task_sep|>",
  "tokenizer_class": "TokenizersBackend"
}