Alignment-Lab-AI commited on
Commit
39782eb
1 Parent(s): 3f8e529

Upload tokenizer

Browse files
special_tokens_map.json CHANGED
@@ -13,13 +13,7 @@
13
  "rstrip": false,
14
  "single_word": false
15
  },
16
- "pad_token": {
17
- "content": "</s>",
18
- "lstrip": false,
19
- "normalized": false,
20
- "rstrip": false,
21
- "single_word": false
22
- },
23
  "unk_token": {
24
  "content": "<unk>",
25
  "lstrip": false,
 
13
  "rstrip": false,
14
  "single_word": false
15
  },
16
+ "pad_token": "</s>",
 
 
 
 
 
 
17
  "unk_token": {
18
  "content": "<unk>",
19
  "lstrip": false,
tokenizer.json CHANGED
@@ -62,6 +62,12 @@
62
  "id": "A",
63
  "type_id": 0
64
  }
 
 
 
 
 
 
65
  }
66
  ],
67
  "pair": [
@@ -77,6 +83,12 @@
77
  "type_id": 0
78
  }
79
  },
 
 
 
 
 
 
80
  {
81
  "SpecialToken": {
82
  "id": "<s>",
@@ -88,9 +100,24 @@
88
  "id": "B",
89
  "type_id": 1
90
  }
 
 
 
 
 
 
91
  }
92
  ],
93
  "special_tokens": {
 
 
 
 
 
 
 
 
 
94
  "<s>": {
95
  "id": "<s>",
96
  "ids": [
 
62
  "id": "A",
63
  "type_id": 0
64
  }
65
+ },
66
+ {
67
+ "SpecialToken": {
68
+ "id": "</s>",
69
+ "type_id": 0
70
+ }
71
  }
72
  ],
73
  "pair": [
 
83
  "type_id": 0
84
  }
85
  },
86
+ {
87
+ "SpecialToken": {
88
+ "id": "</s>",
89
+ "type_id": 0
90
+ }
91
+ },
92
  {
93
  "SpecialToken": {
94
  "id": "<s>",
 
100
  "id": "B",
101
  "type_id": 1
102
  }
103
+ },
104
+ {
105
+ "SpecialToken": {
106
+ "id": "</s>",
107
+ "type_id": 1
108
+ }
109
  }
110
  ],
111
  "special_tokens": {
112
+ "</s>": {
113
+ "id": "</s>",
114
+ "ids": [
115
+ 2
116
+ ],
117
+ "tokens": [
118
+ "</s>"
119
+ ]
120
+ },
121
  "<s>": {
122
  "id": "<s>",
123
  "ids": [
tokenizer_config.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "add_bos_token": true,
3
- "add_eos_token": false,
4
  "added_tokens_decoder": {
5
  "0": {
6
  "content": "<unk>",
 
1
  {
2
  "add_bos_token": true,
3
+ "add_eos_token": true,
4
  "added_tokens_decoder": {
5
  "0": {
6
  "content": "<unk>",