Upload tokenizer

Files changed (3) hide show

tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

tokenizer_config.json CHANGED Viewed

@@ -52,5 +52,6 @@
   "pad_token": "<pad>",
   "sep_token": "</s>",
   "tokenizer_class": "BartTokenizer",
   "unk_token": "<unk>"
 }

   "pad_token": "<pad>",
   "sep_token": "</s>",
   "tokenizer_class": "BartTokenizer",
+  "trim_offsets": true,
   "unk_token": "<unk>"
 }

vocab.json CHANGED Viewed

The diff for this file is too large to render. See raw diff