paruwka commited on
Commit
bbf2aa6
1 Parent(s): b87d5ff

Upload tokenizer

Browse files
Files changed (3) hide show
  1. added_tokens.json +2 -4
  2. tokenizer_config.json +10 -10
  3. vocab.json +3 -1
added_tokens.json CHANGED
@@ -1,6 +1,4 @@
1
  {
2
- "</s>": 67,
3
- "<s>": 66,
4
- "[PAD]": 69,
5
- "[UNK]": 68
6
  }
 
1
  {
2
+ "</s>": 69,
3
+ "<s>": 68
 
 
4
  }
tokenizer_config.json CHANGED
@@ -1,23 +1,23 @@
1
  {
2
  "added_tokens_decoder": {
3
  "66": {
4
- "content": "<s>",
5
- "lstrip": false,
6
  "normalized": false,
7
- "rstrip": false,
8
  "single_word": false,
9
- "special": true
10
  },
11
  "67": {
12
- "content": "</s>",
13
- "lstrip": false,
14
  "normalized": false,
15
- "rstrip": false,
16
  "single_word": false,
17
- "special": true
18
  },
19
  "68": {
20
- "content": "[UNK]",
21
  "lstrip": false,
22
  "normalized": false,
23
  "rstrip": false,
@@ -25,7 +25,7 @@
25
  "special": true
26
  },
27
  "69": {
28
- "content": "[PAD]",
29
  "lstrip": false,
30
  "normalized": false,
31
  "rstrip": false,
 
1
  {
2
  "added_tokens_decoder": {
3
  "66": {
4
+ "content": "[UNK]",
5
+ "lstrip": true,
6
  "normalized": false,
7
+ "rstrip": true,
8
  "single_word": false,
9
+ "special": false
10
  },
11
  "67": {
12
+ "content": "[PAD]",
13
+ "lstrip": true,
14
  "normalized": false,
15
+ "rstrip": true,
16
  "single_word": false,
17
+ "special": false
18
  },
19
  "68": {
20
+ "content": "<s>",
21
  "lstrip": false,
22
  "normalized": false,
23
  "rstrip": false,
 
25
  "special": true
26
  },
27
  "69": {
28
+ "content": "</s>",
29
  "lstrip": false,
30
  "normalized": false,
31
  "rstrip": false,
vocab.json CHANGED
@@ -1,10 +1,11 @@
1
  {
2
- " ": 17,
3
  "&": 12,
4
  "'": 39,
5
  "(": 19,
6
  ")": 8,
7
  "=": 46,
 
 
8
  "a": 36,
9
  "b": 6,
10
  "c": 16,
@@ -31,6 +32,7 @@
31
  "x": 31,
32
  "y": 37,
33
  "z": 33,
 
34
  "«": 20,
35
  "»": 56,
36
  "ß": 42,
 
1
  {
 
2
  "&": 12,
3
  "'": 39,
4
  "(": 19,
5
  ")": 8,
6
  "=": 46,
7
+ "[PAD]": 67,
8
+ "[UNK]": 66,
9
  "a": 36,
10
  "b": 6,
11
  "c": 16,
 
32
  "x": 31,
33
  "y": 37,
34
  "z": 33,
35
+ "|": 17,
36
  "«": 20,
37
  "»": 56,
38
  "ß": 42,