jaswanthrk
/

tokenizer_BPE_32k_vocab_all_data

Model card Files Files and versions Community

jaswanthrk commited on Jan 23

Commit

759ae03

•

1 Parent(s): d6656b9

Upload tokenizer.json

Files changed (1) hide show

tokenizer.json +3 -3

tokenizer.json CHANGED Viewed

@@ -129,11 +129,11 @@
     "model": {
         "type": "BPE",
         "dropout": null,
-        "unk_token": null,
         "continuing_subword_prefix": null,
         "end_of_word_suffix": null,
-        "fuse_unk": false,
-        "byte_fallback": false,
         "vocab": {
             "<unk>": 0,
             "<s>": 1,

     "model": {
         "type": "BPE",
         "dropout": null,
+        "unk_token": "<unk>",
         "continuing_subword_prefix": null,
         "end_of_word_suffix": null,
+        "fuse_unk": true,
+        "byte_fallback": true,
         "vocab": {
             "<unk>": 0,
             "<s>": 1,