Hezam commited on
Commit
1aef45b
1 Parent(s): 2290959

Upload 4 files

Browse files
added_tokens.json CHANGED
@@ -99,9 +99,6 @@
99
  "<extra_id_98>": 32001,
100
  "<extra_id_99>": 32000,
101
  "<extra_id_9>": 32090,
102
- "<s>": 32100,
103
- "article:": 32101,
104
- "category:": 32103,
105
- "classifiction:": 32102,
106
- "generation:": 32104
107
  }
 
99
  "<extra_id_98>": 32001,
100
  "<extra_id_99>": 32000,
101
  "<extra_id_9>": 32090,
102
+ "article: ": 32100,
103
+ "category: ": 32101
 
 
 
104
  }
special_tokens_map.json CHANGED
@@ -1,35 +1,20 @@
1
  {
2
  "additional_special_tokens": [
3
  {
4
- "content": "article:",
5
  "lstrip": false,
6
  "normalized": false,
7
  "rstrip": false,
8
  "single_word": false
9
  },
10
  {
11
- "content": "classifiction:",
12
- "lstrip": false,
13
- "normalized": false,
14
- "rstrip": false,
15
- "single_word": false
16
- },
17
- {
18
- "content": "category:",
19
- "lstrip": false,
20
- "normalized": false,
21
- "rstrip": false,
22
- "single_word": false
23
- },
24
- {
25
- "content": "generation:",
26
  "lstrip": false,
27
  "normalized": false,
28
  "rstrip": false,
29
  "single_word": false
30
  }
31
  ],
32
- "bos_token": "<s>",
33
  "eos_token": {
34
  "content": "</s>",
35
  "lstrip": false,
 
1
  {
2
  "additional_special_tokens": [
3
  {
4
+ "content": "article: ",
5
  "lstrip": false,
6
  "normalized": false,
7
  "rstrip": false,
8
  "single_word": false
9
  },
10
  {
11
+ "content": "category: ",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
12
  "lstrip": false,
13
  "normalized": false,
14
  "rstrip": false,
15
  "single_word": false
16
  }
17
  ],
 
18
  "eos_token": {
19
  "content": "</s>",
20
  "lstrip": false,
tokenizer_config.json CHANGED
@@ -825,7 +825,7 @@
825
  "special": true
826
  },
827
  "32100": {
828
- "content": "<s>",
829
  "lstrip": false,
830
  "normalized": false,
831
  "rstrip": false,
@@ -833,31 +833,7 @@
833
  "special": true
834
  },
835
  "32101": {
836
- "content": "article:",
837
- "lstrip": false,
838
- "normalized": false,
839
- "rstrip": false,
840
- "single_word": false,
841
- "special": true
842
- },
843
- "32102": {
844
- "content": "classifiction:",
845
- "lstrip": false,
846
- "normalized": false,
847
- "rstrip": false,
848
- "single_word": false,
849
- "special": true
850
- },
851
- "32103": {
852
- "content": "category:",
853
- "lstrip": false,
854
- "normalized": false,
855
- "rstrip": false,
856
- "single_word": false,
857
- "special": true
858
- },
859
- "32104": {
860
- "content": "generation:",
861
  "lstrip": false,
862
  "normalized": false,
863
  "rstrip": false,
@@ -866,12 +842,9 @@
866
  }
867
  },
868
  "additional_special_tokens": [
869
- "article:",
870
- "classifiction:",
871
- "category:",
872
- "generation:"
873
  ],
874
- "bos_token": "<s>",
875
  "clean_up_tokenization_spaces": true,
876
  "eos_token": "</s>",
877
  "extra_ids": 100,
 
825
  "special": true
826
  },
827
  "32100": {
828
+ "content": "article: ",
829
  "lstrip": false,
830
  "normalized": false,
831
  "rstrip": false,
 
833
  "special": true
834
  },
835
  "32101": {
836
+ "content": "category: ",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
837
  "lstrip": false,
838
  "normalized": false,
839
  "rstrip": false,
 
842
  }
843
  },
844
  "additional_special_tokens": [
845
+ "article: ",
846
+ "category: "
 
 
847
  ],
 
848
  "clean_up_tokenization_spaces": true,
849
  "eos_token": "</s>",
850
  "extra_ids": 100,