SaulLu commited on
Commit
524ccfa
1 Parent(s): 9ff1d40

set normalized to true

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +1 -1
  2. tokenizer.json +1 -1
special_tokens_map.json CHANGED
@@ -10,6 +10,6 @@
10
  "single_word": false,
11
  "lstrip": true,
12
  "rstrip": false,
13
- "normalized": false
14
  }
15
  }
10
  "single_word": false,
11
  "lstrip": true,
12
  "rstrip": false,
13
+ "normalized": true
14
  }
15
  }
tokenizer.json CHANGED
@@ -46,7 +46,7 @@
46
  "single_word": false,
47
  "lstrip": true,
48
  "rstrip": false,
49
- "normalized": false
50
  }
51
  ],
52
  "normalizer": {
46
  "single_word": false,
47
  "lstrip": true,
48
  "rstrip": false,
49
+ "normalized": true
50
  }
51
  ],
52
  "normalizer": {