itsliupeng commited on
Commit
2b397e5
1 Parent(s): d0b4d06

update tokenizer.json to fix missing space problem

Browse files
Files changed (1) hide show
  1. tokenizer.json +6 -6
tokenizer.json CHANGED
@@ -2116,17 +2116,17 @@
2116
  },
2117
  "content": " "
2118
  },
2119
- {
2120
- "type": "Strip",
2121
- "content": " ",
2122
- "start": 1,
2123
- "stop": 0
2124
- },
2125
  {
2126
  "type": "ByteFallback"
2127
  },
2128
  {
2129
  "type": "Fuse"
 
 
 
 
 
 
2130
  }
2131
  ]
2132
  },
 
2116
  },
2117
  "content": " "
2118
  },
 
 
 
 
 
 
2119
  {
2120
  "type": "ByteFallback"
2121
  },
2122
  {
2123
  "type": "Fuse"
2124
+ },
2125
+ {
2126
+ "type": "Strip",
2127
+ "content": " ",
2128
+ "start": 1,
2129
+ "stop": 0
2130
  }
2131
  ]
2132
  },