sdadas commited on
Commit
3fc9db5
1 Parent(s): 705ba73

Upload tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +1 -1
tokenizer.json CHANGED
@@ -67,7 +67,7 @@
67
  "pre_tokenizer": {
68
  "type": "Metaspace",
69
  "replacement": "▁",
70
- "add_prefix_space": true
71
  },
72
  "post_processor": {
73
  "type": "RobertaProcessing",
 
67
  "pre_tokenizer": {
68
  "type": "Metaspace",
69
  "replacement": "▁",
70
+ "prepend_scheme": "never"
71
  },
72
  "post_processor": {
73
  "type": "RobertaProcessing",