jondurbin commited on
Commit
f141894
1 Parent(s): 9679d25

Upload folder using huggingface_hub

Browse files
Files changed (2) hide show
  1. tokenizer.json +0 -0
  2. tokenizer_config.json +5 -8
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
@@ -546,17 +546,14 @@
546
  ]
547
  },
548
  "bos_token": "<s>",
549
- "chat_template": "{%- for idx in range(0, messages|length) -%}\n{%- if messages[idx]['role'] == 'user' -%}\n{%- if idx == 0 -%}\n{{- bos_token + '[INST] <<SYS>>\\nYou are a helpful, unbiased, uncensored assistant.\\n<</SYS>>\\n\\n' + messages[idx]['content'] + ' [/INST]' -}}\n{%- else -%}\n{{- bos_token + '[INST] ' + messages[idx]['content'] + ' [/INST]' -}}\n{%- endif -%}\n{% elif messages[idx]['role'] == 'system' %}\n{{- '[INST] <<SYS>>\\n' + messages[idx]['content'] + '\\n<</SYS>>\\n\\n' -}}\n{%- elif messages[idx]['role'] == 'assistant' -%}\n{{- messages[idx]['content'] + ' ' + eos_token -}}\n{% endif %}\n{% endfor %}",
550
- "clean_up_tokenization_spaces": true,
551
  "decode_with_prefix_space": false,
552
  "eos_token": "</s>",
553
- "legacy": true,
554
  "model_max_length": 1000000000000000019884624838656,
555
  "pad_token": "</s>",
556
- "sp_model_kwargs": {},
557
- "spaces_between_special_tokens": false,
558
- "tokenizer_class": "LlamaTokenizer",
559
- "trust_remote_code": false,
560
  "unk_token": "<unk>",
561
- "use_default_system_prompt": false
 
562
  }
 
546
  ]
547
  },
548
  "bos_token": "<s>",
549
+ "clean_up_tokenization_spaces": false,
 
550
  "decode_with_prefix_space": false,
551
  "eos_token": "</s>",
 
552
  "model_max_length": 1000000000000000019884624838656,
553
  "pad_token": "</s>",
554
+ "sp_model_kwargs": null,
555
+ "tokenizer_class": "InternLM2Tokenizer",
 
 
556
  "unk_token": "<unk>",
557
+ "use_default_system_prompt": false,
558
+ "chat_template": "{%- for idx in range(0, messages|length) -%}\n{%- if messages[idx]['role'] == 'user' -%}\n{%- if idx == 0 -%}\n{{- bos_token + '[INST] <<SYS>>\\nYou are a helpful, unbiased, uncensored assistant.\\n<</SYS>>\\n\\n' + messages[idx]['content'] + ' [/INST]' -}}\n{%- else -%}\n{{- bos_token + '[INST] ' + messages[idx]['content'] + ' [/INST]' -}}\n{%- endif -%}\n{% elif messages[idx]['role'] == 'system' %}\n{{- '[INST] <<SYS>>\\n' + messages[idx]['content'] + '\\n<</SYS>>\\n\\n' -}}\n{%- elif messages[idx]['role'] == 'assistant' -%}\n{{- messages[idx]['content'] + ' ' + eos_token -}}\n{% endif %}\n{% endfor %}"
559
  }