rippertnt commited on
Commit
7d4edd6
1 Parent(s): fe5bfbe

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +1 -0
  2. tokenizer_config.json +2 -2
tokenizer.json CHANGED
@@ -2098,6 +2098,7 @@
2098
  "continuing_subword_prefix": null,
2099
  "end_of_word_suffix": null,
2100
  "fuse_unk": false,
 
2101
  "vocab": {
2102
  "<s>": 0,
2103
  "</s>": 1,
 
2098
  "continuing_subword_prefix": null,
2099
  "end_of_word_suffix": null,
2100
  "fuse_unk": false,
2101
+ "byte_fallback": false,
2102
  "vocab": {
2103
  "<s>": 0,
2104
  "</s>": 1,
tokenizer_config.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "name_or_path": "circulus/canvers-ko2en-v1",
3
- "special_tokens_map_file": "/home/circulus/.cache/huggingface/transformers/a87d2ed77831bb40ce806a97c04126addf5ecc82b3e23ecf916b2a4acdb9c29a.c23d5e62137984cf842a885705037b25b156747d145406702932d5f5d5e7c88e",
4
  "tokenizer_class": "PreTrainedTokenizerFast"
5
  }
 
1
  {
2
+ "clean_up_tokenization_spaces": true,
3
+ "model_max_length": 1000000000000000019884624838656,
4
  "tokenizer_class": "PreTrainedTokenizerFast"
5
  }