JFernandoGRE commited on
Commit
f27d7b8
1 Parent(s): 260bc42

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer.json +1 -0
tokenizer.json CHANGED
@@ -155,6 +155,7 @@
155
  "end_of_word_suffix": null,
156
  "fuse_unk": false,
157
  "byte_fallback": false,
 
158
  "vocab": {
159
  ">>TITLE<<": 0,
160
  ">>ABSTRACT<<": 1,
 
155
  "end_of_word_suffix": null,
156
  "fuse_unk": false,
157
  "byte_fallback": false,
158
+ "ignore_merges": false,
159
  "vocab": {
160
  ">>TITLE<<": 0,
161
  ">>ABSTRACT<<": 1,