KoichiYasuoka commited on
Commit
f81df7a
1 Parent(s): baf3467

tokenizer improved

Browse files
Files changed (1) hide show
  1. tokenizer.json +3 -1
tokenizer.json CHANGED
@@ -10846,6 +10846,7 @@
10846
  "pre_tokenizer": {
10847
  "type": "Metaspace",
10848
  "replacement": "▁",
 
10849
  "prepend_scheme": "never",
10850
  "split": true
10851
  },
@@ -10927,6 +10928,7 @@
10927
  "decoder": {
10928
  "type": "Metaspace",
10929
  "replacement": "▁",
 
10930
  "prepend_scheme": "never",
10931
  "split": true
10932
  },
@@ -210937,4 +210939,4 @@
210937
  ],
210938
  "byte_fallback": false
210939
  }
210940
- }
 
10846
  "pre_tokenizer": {
10847
  "type": "Metaspace",
10848
  "replacement": "▁",
10849
+ "add_prefix_space": false,
10850
  "prepend_scheme": "never",
10851
  "split": true
10852
  },
 
10928
  "decoder": {
10929
  "type": "Metaspace",
10930
  "replacement": "▁",
10931
+ "add_prefix_space": false,
10932
  "prepend_scheme": "never",
10933
  "split": true
10934
  },
 
210939
  ],
210940
  "byte_fallback": false
210941
  }
210942
+ }