silvainrichou commited on
Commit
d8c39ad
1 Parent(s): da8a543

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +2 -2
  2. tokenizer_config.json +16 -0
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:a2d83c8e6a960de3d36db939e71a1857057ea89ce784a795979fe885457c3e9c
3
- size 17478274
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e4a00d8ec980161dcec47ca69be226c7fcb639b1440a977ad934a40cec16ca41
3
+ size 17478284
tokenizer_config.json CHANGED
@@ -49,6 +49,22 @@
49
  "rstrip": false,
50
  "single_word": false,
51
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
52
  }
53
  },
54
  "additional_special_tokens": [
 
49
  "rstrip": false,
50
  "single_word": false,
51
  "special": true
52
+ },
53
+ "204": {
54
+ "content": "<s>",
55
+ "lstrip": false,
56
+ "normalized": false,
57
+ "rstrip": false,
58
+ "single_word": false,
59
+ "special": true
60
+ },
61
+ "213": {
62
+ "content": "</s>",
63
+ "lstrip": false,
64
+ "normalized": false,
65
+ "rstrip": false,
66
+ "single_word": false,
67
+ "special": true
68
  }
69
  },
70
  "additional_special_tokens": [