thomasw21 commited on
Commit
d56c744
1 Parent(s): 8943cc8

Fix tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -1 +1 @@
1
- {"unk_token": "<unk>", "eos_token": "</s>", "bos_token": "<s>", "pad_token": "<pad>", "special_tokens_map_file": "/Users/thomas/.cache/huggingface/transformers/9b8b2f4cb97dda0753c9b7213ca10bae9674703a4c64f786341b96a260d44985.9d6cd81ef646692fb1c169a880161ea1cb95f49694f220aced9b704b457e51dd", "name_or_path": "bigscience-catalogue-data-dev/byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles", "tokenizer_class": "PreTrainedTokenizerFast"}
 
1
+ {"unk_token": "<unk>", "eos_token": "</s>", "bos_token": "<s>", "pad_token": "<pad>", "name_or_path": "bigscience/tokenizer", "special_tokens_map_file": null, "tokenizer_class": "PreTrainedTokenizerFast"}