Panchovix commited on
Commit
72b9ee1
1 Parent(s): deed8ca

Upload 4 files

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +24 -0
  2. tokenizer.json +9 -12
special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": true,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": true,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "<unk>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": true,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
tokenizer.json CHANGED
@@ -9,7 +9,7 @@
9
  "single_word": false,
10
  "lstrip": false,
11
  "rstrip": false,
12
- "normalized": true,
13
  "special": true
14
  },
15
  {
@@ -27,16 +27,7 @@
27
  "single_word": false,
28
  "lstrip": false,
29
  "rstrip": false,
30
- "normalized": false,
31
- "special": true
32
- },
33
- {
34
- "id": 32000,
35
- "content": "[PAD]",
36
- "single_word": false,
37
- "lstrip": false,
38
- "rstrip": false,
39
- "normalized": false,
40
  "special": true
41
  }
42
  ],
@@ -86,10 +77,16 @@
86
  "type_id": 0
87
  }
88
  },
 
 
 
 
 
 
89
  {
90
  "Sequence": {
91
  "id": "B",
92
- "type_id": 0
93
  }
94
  }
95
  ],
 
9
  "single_word": false,
10
  "lstrip": false,
11
  "rstrip": false,
12
+ "normalized": false,
13
  "special": true
14
  },
15
  {
 
27
  "single_word": false,
28
  "lstrip": false,
29
  "rstrip": false,
30
+ "normalized": true,
 
 
 
 
 
 
 
 
 
31
  "special": true
32
  }
33
  ],
 
77
  "type_id": 0
78
  }
79
  },
80
+ {
81
+ "SpecialToken": {
82
+ "id": "<s>",
83
+ "type_id": 1
84
+ }
85
+ },
86
  {
87
  "Sequence": {
88
  "id": "B",
89
+ "type_id": 1
90
  }
91
  }
92
  ],