lalital commited on
Commit
04ff256
1 Parent(s): fd76d90

Change space token from <th_roberta_space_token> to <_>

Browse files
Files changed (2) hide show
  1. sentencepiece.bpe.vocab +1 -1
  2. tokenizer_config.json +2 -2
sentencepiece.bpe.vocab CHANGED
@@ -2,7 +2,7 @@
2
  <s> 0
3
  </s> 0
4
  <mask> 0
5
- <th_roberta_space_token> 0
6
  า -1.82049
7
  ▁ -3.88454
8
  . -4.13722
 
2
  <s> 0
3
  </s> 0
4
  <mask> 0
5
+ <_> 0
6
  า -1.82049
7
  ▁ -3.88454
8
  . -4.13722
tokenizer_config.json CHANGED
@@ -10,6 +10,6 @@
10
  "additional_special_tokens": [
11
  "<s>NOTUSED",
12
  "</s>NOTUSED",
13
- "<th_roberta_space_token>"
14
  ]
15
- }
 
10
  "additional_special_tokens": [
11
  "<s>NOTUSED",
12
  "</s>NOTUSED",
13
+ "<_>"
14
  ]
15
+ }