hans00 commited on
Commit
9ba62f0
·
verified ·
1 Parent(s): 1900559

Fix add blank

Browse files
Files changed (1) hide show
  1. tokenizer.json +1 -1
tokenizer.json CHANGED
@@ -28,7 +28,7 @@
28
  {
29
  "type": "Replace",
30
  "pattern": {
31
- "Regex": "(?:\\[SEP\\]|\\b)"
32
  },
33
  "content": "_"
34
  },
 
28
  {
29
  "type": "Replace",
30
  "pattern": {
31
+ "Regex": "(?:\\[SEP\\]|^|$)"
32
  },
33
  "content": "_"
34
  },