floriangardin commited on
Commit
937ae1f
·
1 Parent(s): 7d37b58

Update tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +1 -1
tokenizer.json CHANGED
@@ -19,7 +19,7 @@
19
  },
20
  "post_processor": null,
21
  "decoder": {
22
- "type": "Strip"
23
  },
24
  "model": {
25
  "type": "WordLevel",
 
19
  },
20
  "post_processor": null,
21
  "decoder": {
22
+ "type": "WordPiece"
23
  },
24
  "model": {
25
  "type": "WordLevel",