sdadas commited on
Commit
2f5cff0
1 Parent(s): 638c0c8

Upload tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +2 -1
tokenizer.json CHANGED
@@ -67,7 +67,8 @@
67
  "pre_tokenizer": {
68
  "type": "Metaspace",
69
  "replacement": "▁",
70
- "add_prefix_space": true
 
71
  },
72
  "post_processor": {
73
  "type": "RobertaProcessing",
 
67
  "pre_tokenizer": {
68
  "type": "Metaspace",
69
  "replacement": "▁",
70
+ "add_prefix_space": true,
71
+ "prepend_scheme": "never"
72
  },
73
  "post_processor": {
74
  "type": "RobertaProcessing",