model commit
Browse files- .gitattributes +1 -4
- README.md +55 -1
- added_tokens.json +237 -1
- config.json +10 -3
- merges.txt +1 -1
- pytorch_model.bin +0 -3
- special_tokens_map.json +15 -1
- tokenizer.json +0 -0
- tokenizer_config.json +64 -1
.gitattributes
CHANGED
@@ -2,13 +2,11 @@
|
|
2 |
*.arrow filter=lfs diff=lfs merge=lfs -text
|
3 |
*.bin filter=lfs diff=lfs merge=lfs -text
|
4 |
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
6 |
*.ftz filter=lfs diff=lfs merge=lfs -text
|
7 |
*.gz filter=lfs diff=lfs merge=lfs -text
|
8 |
*.h5 filter=lfs diff=lfs merge=lfs -text
|
9 |
*.joblib filter=lfs diff=lfs merge=lfs -text
|
10 |
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
12 |
*.model filter=lfs diff=lfs merge=lfs -text
|
13 |
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
14 |
*.npy filter=lfs diff=lfs merge=lfs -text
|
@@ -22,10 +20,8 @@
|
|
22 |
*.pt filter=lfs diff=lfs merge=lfs -text
|
23 |
*.pth filter=lfs diff=lfs merge=lfs -text
|
24 |
*.rar filter=lfs diff=lfs merge=lfs -text
|
25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
26 |
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
27 |
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
29 |
*.tflite filter=lfs diff=lfs merge=lfs -text
|
30 |
*.tgz filter=lfs diff=lfs merge=lfs -text
|
31 |
*.wasm filter=lfs diff=lfs merge=lfs -text
|
@@ -33,3 +29,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
2 |
*.arrow filter=lfs diff=lfs merge=lfs -text
|
3 |
*.bin filter=lfs diff=lfs merge=lfs -text
|
4 |
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
|
|
5 |
*.ftz filter=lfs diff=lfs merge=lfs -text
|
6 |
*.gz filter=lfs diff=lfs merge=lfs -text
|
7 |
*.h5 filter=lfs diff=lfs merge=lfs -text
|
8 |
*.joblib filter=lfs diff=lfs merge=lfs -text
|
9 |
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
|
|
10 |
*.model filter=lfs diff=lfs merge=lfs -text
|
11 |
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
12 |
*.npy filter=lfs diff=lfs merge=lfs -text
|
|
|
20 |
*.pt filter=lfs diff=lfs merge=lfs -text
|
21 |
*.pth filter=lfs diff=lfs merge=lfs -text
|
22 |
*.rar filter=lfs diff=lfs merge=lfs -text
|
|
|
23 |
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
24 |
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
|
|
25 |
*.tflite filter=lfs diff=lfs merge=lfs -text
|
26 |
*.tgz filter=lfs diff=lfs merge=lfs -text
|
27 |
*.wasm filter=lfs diff=lfs merge=lfs -text
|
|
|
29 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
30 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
31 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
32 |
+
model.safetensors filter=lfs diff=lfs merge=lfs -text
|
README.md
CHANGED
@@ -1,4 +1,58 @@
|
|
1 |
---
|
2 |
-
|
|
|
|
|
|
|
|
|
3 |
---
|
4 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
---
|
2 |
+
language: en
|
3 |
+
license: apache-2.0
|
4 |
+
datasets: climatebert/environmental_claims
|
5 |
+
tags:
|
6 |
+
- Env Claims
|
7 |
---
|
8 |
|
9 |
+
# Model Card for environmental-claims
|
10 |
+
|
11 |
+
## Model Description
|
12 |
+
|
13 |
+
|
14 |
+
|
15 |
+
## Climate Performance Model Card
|
16 |
+
|
17 |
+
| environmental-claims | |
|
18 |
+
|--------------------------------------------------------------------------|----------------|
|
19 |
+
| 1. Is the resulting model publicly available? | Yes |
|
20 |
+
| 2. How much time does the training of the final model take? | < 5 min |
|
21 |
+
| 3. How much time did all experiments take (incl. hyperparameter search)? | 60 hours |
|
22 |
+
| 4. What was the power of GPU and CPU? | 0.3 kW |
|
23 |
+
| 5. At which geo location were the computations performed? | Switzerland |
|
24 |
+
| 6. What was the energy mix at the geo location? | 89 gCO2eq/kWh |
|
25 |
+
| 7. How much CO2eq was emitted to train the final model? | 2.2 g |
|
26 |
+
| 8. How much CO2eq was emitted for all experiments? | 1.6 kg |
|
27 |
+
| 9. What is the average CO2eq emission for the inference of one sample? | 0.0067 mg |
|
28 |
+
| 10. Which positive environmental impact can be expected from this work? | This work can help detect and evaluate environmental claims and thus have a positive impact on the environment in the future. |
|
29 |
+
| 11. Comments | - |
|
30 |
+
|
31 |
+
## Citation Information
|
32 |
+
|
33 |
+
|
34 |
+
## How to Get Started With the Model
|
35 |
+
|
36 |
+
You can use the model with a pipeline for text classification:
|
37 |
+
|
38 |
+
```python
|
39 |
+
from transformers import AutoModelForSequenceClassification, AutoTokenizer, pipeline
|
40 |
+
from transformers.pipelines.pt_utils import KeyDataset
|
41 |
+
import datasets
|
42 |
+
from tqdm.auto import tqdm
|
43 |
+
|
44 |
+
dataset_name = "climatebert/environmental_claims"
|
45 |
+
|
46 |
+
|
47 |
+
|
48 |
+
dataset = datasets.load_dataset(dataset_name, split="test")
|
49 |
+
|
50 |
+
model = AutoModelForSequenceClassification.from_pretrained(model_name)
|
51 |
+
tokenizer = AutoTokenizer.from_pretrained(model_name, max_len=512)
|
52 |
+
|
53 |
+
pipe = pipeline("text-classification", model=model, tokenizer=tokenizer, device=0)
|
54 |
+
|
55 |
+
# See https://huggingface.co/docs/transformers/main_classes/pipelines#transformers.pipeline
|
56 |
+
for out in tqdm(pipe(KeyDataset(dataset, "text"), padding=True, truncation=True)):
|
57 |
+
print(out)
|
58 |
+
```
|
added_tokens.json
CHANGED
@@ -1 +1,237 @@
|
|
1 |
-
{
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"+/-": 50403,
|
3 |
+
"2021": 50328,
|
4 |
+
"2030": 50417,
|
5 |
+
"2050": 50487,
|
6 |
+
"CH4": 50354,
|
7 |
+
"CO2": 50265,
|
8 |
+
"Committee": 50410,
|
9 |
+
"GHG": 50397,
|
10 |
+
"N2O": 50382,
|
11 |
+
"achieve": 50437,
|
12 |
+
"across": 50277,
|
13 |
+
"activities": 50317,
|
14 |
+
"adaptation": 50377,
|
15 |
+
"addition": 50303,
|
16 |
+
"additional": 50444,
|
17 |
+
"affect": 50438,
|
18 |
+
"agreement": 50464,
|
19 |
+
"agricultural": 50395,
|
20 |
+
"already": 50435,
|
21 |
+
"analyses": 50485,
|
22 |
+
"annual": 50287,
|
23 |
+
"applied": 50360,
|
24 |
+
"approach": 50290,
|
25 |
+
"areas": 50275,
|
26 |
+
"assess": 50430,
|
27 |
+
"assessment": 50345,
|
28 |
+
"atmosphere": 50341,
|
29 |
+
"atmospheric": 50310,
|
30 |
+
"basis": 50372,
|
31 |
+
"become": 50394,
|
32 |
+
"benefits": 50402,
|
33 |
+
"biomass": 50314,
|
34 |
+
"capture": 50379,
|
35 |
+
"caused": 50389,
|
36 |
+
"challenges": 50404,
|
37 |
+
"characteristics": 50420,
|
38 |
+
"climatic": 50319,
|
39 |
+
"coastal": 50393,
|
40 |
+
"combined": 50451,
|
41 |
+
"communities": 50318,
|
42 |
+
"companies": 50312,
|
43 |
+
"compared": 50282,
|
44 |
+
"composition": 50418,
|
45 |
+
"concentration": 50357,
|
46 |
+
"concentrations": 50366,
|
47 |
+
"conditions": 50272,
|
48 |
+
"conducted": 50439,
|
49 |
+
"conservation": 50458,
|
50 |
+
"considered": 50339,
|
51 |
+
"consistent": 50494,
|
52 |
+
"construction": 50445,
|
53 |
+
"consumption": 50307,
|
54 |
+
"contribute": 50498,
|
55 |
+
"contribution": 50468,
|
56 |
+
"costs": 50326,
|
57 |
+
"countries": 50283,
|
58 |
+
"crisis": 50499,
|
59 |
+
"customers": 50358,
|
60 |
+
"decades": 50495,
|
61 |
+
"decision": 50452,
|
62 |
+
"decrease": 50367,
|
63 |
+
"decreased": 50396,
|
64 |
+
"degrees": 50276,
|
65 |
+
"delta": 50433,
|
66 |
+
"determine": 50480,
|
67 |
+
"determined": 50455,
|
68 |
+
"developing": 50412,
|
69 |
+
"differences": 50375,
|
70 |
+
"dioxide": 50346,
|
71 |
+
"distribution": 50296,
|
72 |
+
"diversity": 50431,
|
73 |
+
"drought": 50336,
|
74 |
+
"dynamics": 50390,
|
75 |
+
"ecological": 50401,
|
76 |
+
"economy": 50338,
|
77 |
+
"ecosystem": 50333,
|
78 |
+
"ecosystems": 50384,
|
79 |
+
"efforts": 50392,
|
80 |
+
"electricity": 50315,
|
81 |
+
"emission": 50279,
|
82 |
+
"emissions": 50266,
|
83 |
+
"employees": 50419,
|
84 |
+
"ensure": 50449,
|
85 |
+
"environmental": 50269,
|
86 |
+
"estimate": 50459,
|
87 |
+
"estimated": 50362,
|
88 |
+
"estimates": 50408,
|
89 |
+
"factors": 50292,
|
90 |
+
"findings": 50492,
|
91 |
+
"flood": 50363,
|
92 |
+
"flux": 50476,
|
93 |
+
"fluxes": 50491,
|
94 |
+
"following": 50369,
|
95 |
+
"forests": 50406,
|
96 |
+
"fossil": 50348,
|
97 |
+
"fuels": 50424,
|
98 |
+
"further": 50301,
|
99 |
+
"gases": 50471,
|
100 |
+
"greater": 50368,
|
101 |
+
"greenhouse": 50289,
|
102 |
+
"however": 50454,
|
103 |
+
"hydrogen": 50376,
|
104 |
+
"identify": 50460,
|
105 |
+
"impacts": 50281,
|
106 |
+
"importance": 50414,
|
107 |
+
"included": 50434,
|
108 |
+
"increase": 50271,
|
109 |
+
"increased": 50274,
|
110 |
+
"increases": 50361,
|
111 |
+
"indicate": 50388,
|
112 |
+
"industry": 50306,
|
113 |
+
"influence": 50329,
|
114 |
+
"infrastructure": 50425,
|
115 |
+
"integrated": 50483,
|
116 |
+
"investigated": 50385,
|
117 |
+
"investment": 50321,
|
118 |
+
"investments": 50466,
|
119 |
+
"least": 50429,
|
120 |
+
"losses": 50462,
|
121 |
+
"mainly": 50399,
|
122 |
+
"materials": 50398,
|
123 |
+
"means": 50486,
|
124 |
+
"measured": 50364,
|
125 |
+
"measurements": 50422,
|
126 |
+
"methane": 50359,
|
127 |
+
"methods": 50378,
|
128 |
+
"mitigation": 50457,
|
129 |
+
"moisture": 50493,
|
130 |
+
"monitoring": 50465,
|
131 |
+
"nitrogen": 50405,
|
132 |
+
"northern": 50446,
|
133 |
+
"observations": 50461,
|
134 |
+
"observed": 50293,
|
135 |
+
"obtained": 50391,
|
136 |
+
"ocean": 50347,
|
137 |
+
"operating": 50440,
|
138 |
+
"operations": 50374,
|
139 |
+
"opportunities": 50469,
|
140 |
+
"overall": 50426,
|
141 |
+
"pandemic": 50490,
|
142 |
+
"parameters": 50383,
|
143 |
+
"particular": 50413,
|
144 |
+
"patterns": 50349,
|
145 |
+
"performed": 50497,
|
146 |
+
"periods": 50432,
|
147 |
+
"planning": 50473,
|
148 |
+
"plans": 50423,
|
149 |
+
"plants": 50313,
|
150 |
+
"policies": 50355,
|
151 |
+
"pollution": 50467,
|
152 |
+
"populations": 50441,
|
153 |
+
"possible": 50332,
|
154 |
+
"potential": 50273,
|
155 |
+
"practices": 50453,
|
156 |
+
"precipitation": 50280,
|
157 |
+
"presented": 50428,
|
158 |
+
"previous": 50482,
|
159 |
+
"processes": 50291,
|
160 |
+
"productivity": 50463,
|
161 |
+
"proposed": 50370,
|
162 |
+
"provide": 50285,
|
163 |
+
"provides": 50373,
|
164 |
+
"rainfall": 50323,
|
165 |
+
"ratio": 50416,
|
166 |
+
"recovery": 50450,
|
167 |
+
"reduce": 50288,
|
168 |
+
"reduced": 50327,
|
169 |
+
"reducing": 50381,
|
170 |
+
"reduction": 50286,
|
171 |
+
"regional": 50308,
|
172 |
+
"regions": 50302,
|
173 |
+
"relationship": 50400,
|
174 |
+
"relatively": 50484,
|
175 |
+
"renewable": 50294,
|
176 |
+
"requirements": 50477,
|
177 |
+
"respectively": 50316,
|
178 |
+
"responses": 50427,
|
179 |
+
"resulting": 50456,
|
180 |
+
"risks": 50309,
|
181 |
+
"scenario": 50421,
|
182 |
+
"scenarios": 50334,
|
183 |
+
"seasonal": 50411,
|
184 |
+
"sediment": 50475,
|
185 |
+
"several": 50342,
|
186 |
+
"shares": 50474,
|
187 |
+
"showed": 50304,
|
188 |
+
"significantly": 50299,
|
189 |
+
"simulations": 50470,
|
190 |
+
"snow": 50496,
|
191 |
+
"soil": 50270,
|
192 |
+
"soils": 50448,
|
193 |
+
"solar": 50320,
|
194 |
+
"solutions": 50351,
|
195 |
+
"sources": 50331,
|
196 |
+
"southern": 50481,
|
197 |
+
"spatial": 50322,
|
198 |
+
"statements": 50472,
|
199 |
+
"strategies": 50387,
|
200 |
+
"strategy": 50353,
|
201 |
+
"structure": 50337,
|
202 |
+
"studied": 50443,
|
203 |
+
"studies": 50297,
|
204 |
+
"summer": 50335,
|
205 |
+
"supply": 50311,
|
206 |
+
"sustainability": 50325,
|
207 |
+
"sustainable": 50284,
|
208 |
+
"systems": 50278,
|
209 |
+
"targets": 50436,
|
210 |
+
"technologies": 50343,
|
211 |
+
"temperature": 50268,
|
212 |
+
"temperatures": 50295,
|
213 |
+
"temporal": 50479,
|
214 |
+
"thermal": 50365,
|
215 |
+
"towards": 50409,
|
216 |
+
"transition": 50344,
|
217 |
+
"transport": 50352,
|
218 |
+
"trees": 50478,
|
219 |
+
"trend": 50407,
|
220 |
+
"trends": 50371,
|
221 |
+
"tropical": 50415,
|
222 |
+
"uncertainty": 50489,
|
223 |
+
"understanding": 50356,
|
224 |
+
"variability": 50298,
|
225 |
+
"variables": 50447,
|
226 |
+
"variation": 50380,
|
227 |
+
"variations": 50442,
|
228 |
+
"various": 50340,
|
229 |
+
"vegetation": 50330,
|
230 |
+
"waste": 50350,
|
231 |
+
"yield": 50386,
|
232 |
+
"–": 50300,
|
233 |
+
"’": 50267,
|
234 |
+
"“": 50305,
|
235 |
+
"”": 50488,
|
236 |
+
"•": 50324
|
237 |
+
}
|
config.json
CHANGED
@@ -1,5 +1,5 @@
|
|
1 |
{
|
2 |
-
"_name_or_path": "climatebert
|
3 |
"architectures": [
|
4 |
"RobertaForSequenceClassification"
|
5 |
],
|
@@ -7,12 +7,19 @@
|
|
7 |
"bos_token_id": 0,
|
8 |
"classifier_dropout": null,
|
9 |
"eos_token_id": 2,
|
10 |
-
"gradient_checkpointing": true,
|
11 |
"hidden_act": "gelu",
|
12 |
"hidden_dropout_prob": 0.1,
|
13 |
"hidden_size": 768,
|
|
|
|
|
|
|
|
|
14 |
"initializer_range": 0.02,
|
15 |
"intermediate_size": 3072,
|
|
|
|
|
|
|
|
|
16 |
"layer_norm_eps": 1e-05,
|
17 |
"max_position_embeddings": 514,
|
18 |
"model_type": "roberta",
|
@@ -22,7 +29,7 @@
|
|
22 |
"position_embedding_type": "absolute",
|
23 |
"problem_type": "single_label_classification",
|
24 |
"torch_dtype": "float32",
|
25 |
-
"transformers_version": "4.
|
26 |
"type_vocab_size": 1,
|
27 |
"use_cache": true,
|
28 |
"vocab_size": 50500
|
|
|
1 |
{
|
2 |
+
"_name_or_path": "./climatebert-environmental-claims",
|
3 |
"architectures": [
|
4 |
"RobertaForSequenceClassification"
|
5 |
],
|
|
|
7 |
"bos_token_id": 0,
|
8 |
"classifier_dropout": null,
|
9 |
"eos_token_id": 2,
|
|
|
10 |
"hidden_act": "gelu",
|
11 |
"hidden_dropout_prob": 0.1,
|
12 |
"hidden_size": 768,
|
13 |
+
"id2label": {
|
14 |
+
"0": "no",
|
15 |
+
"1": "yes"
|
16 |
+
},
|
17 |
"initializer_range": 0.02,
|
18 |
"intermediate_size": 3072,
|
19 |
+
"label2id": {
|
20 |
+
"no": 0,
|
21 |
+
"yes": 1
|
22 |
+
},
|
23 |
"layer_norm_eps": 1e-05,
|
24 |
"max_position_embeddings": 514,
|
25 |
"model_type": "roberta",
|
|
|
29 |
"position_embedding_type": "absolute",
|
30 |
"problem_type": "single_label_classification",
|
31 |
"torch_dtype": "float32",
|
32 |
+
"transformers_version": "4.29.2",
|
33 |
"type_vocab_size": 1,
|
34 |
"use_cache": true,
|
35 |
"vocab_size": 50500
|
merges.txt
CHANGED
@@ -1,4 +1,4 @@
|
|
1 |
-
#version: 0.2
|
2 |
Ġ t
|
3 |
Ġ a
|
4 |
h e
|
|
|
1 |
+
#version: 0.2
|
2 |
Ġ t
|
3 |
Ġ a
|
4 |
h e
|
pytorch_model.bin
DELETED
@@ -1,3 +0,0 @@
|
|
1 |
-
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:c8c12117c7110ab8b8577d8b4b276ceda2e97604312d750ce81882d55c35c1d4
|
3 |
-
size 329250921
|
|
|
|
|
|
|
|
special_tokens_map.json
CHANGED
@@ -1 +1,15 @@
|
|
1 |
-
{
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"bos_token": "<s>",
|
3 |
+
"cls_token": "<s>",
|
4 |
+
"eos_token": "</s>",
|
5 |
+
"mask_token": {
|
6 |
+
"content": "<mask>",
|
7 |
+
"lstrip": true,
|
8 |
+
"normalized": false,
|
9 |
+
"rstrip": false,
|
10 |
+
"single_word": false
|
11 |
+
},
|
12 |
+
"pad_token": "<pad>",
|
13 |
+
"sep_token": "</s>",
|
14 |
+
"unk_token": "<unk>"
|
15 |
+
}
|
tokenizer.json
CHANGED
The diff for this file is too large to render.
See raw diff
|
|
tokenizer_config.json
CHANGED
@@ -1 +1,64 @@
|
|
1 |
-
{
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"add_prefix_space": false,
|
3 |
+
"bos_token": {
|
4 |
+
"__type": "AddedToken",
|
5 |
+
"content": "<s>",
|
6 |
+
"lstrip": false,
|
7 |
+
"normalized": true,
|
8 |
+
"rstrip": false,
|
9 |
+
"single_word": false
|
10 |
+
},
|
11 |
+
"clean_up_tokenization_spaces": true,
|
12 |
+
"cls_token": {
|
13 |
+
"__type": "AddedToken",
|
14 |
+
"content": "<s>",
|
15 |
+
"lstrip": false,
|
16 |
+
"normalized": true,
|
17 |
+
"rstrip": false,
|
18 |
+
"single_word": false
|
19 |
+
},
|
20 |
+
"eos_token": {
|
21 |
+
"__type": "AddedToken",
|
22 |
+
"content": "</s>",
|
23 |
+
"lstrip": false,
|
24 |
+
"normalized": true,
|
25 |
+
"rstrip": false,
|
26 |
+
"single_word": false
|
27 |
+
},
|
28 |
+
"errors": "replace",
|
29 |
+
"mask_token": {
|
30 |
+
"__type": "AddedToken",
|
31 |
+
"content": "<mask>",
|
32 |
+
"lstrip": true,
|
33 |
+
"normalized": true,
|
34 |
+
"rstrip": false,
|
35 |
+
"single_word": false
|
36 |
+
},
|
37 |
+
"model_max_length": 512,
|
38 |
+
"pad_token": {
|
39 |
+
"__type": "AddedToken",
|
40 |
+
"content": "<pad>",
|
41 |
+
"lstrip": false,
|
42 |
+
"normalized": true,
|
43 |
+
"rstrip": false,
|
44 |
+
"single_word": false
|
45 |
+
},
|
46 |
+
"sep_token": {
|
47 |
+
"__type": "AddedToken",
|
48 |
+
"content": "</s>",
|
49 |
+
"lstrip": false,
|
50 |
+
"normalized": true,
|
51 |
+
"rstrip": false,
|
52 |
+
"single_word": false
|
53 |
+
},
|
54 |
+
"tokenizer_class": "RobertaTokenizer",
|
55 |
+
"trim_offsets": true,
|
56 |
+
"unk_token": {
|
57 |
+
"__type": "AddedToken",
|
58 |
+
"content": "<unk>",
|
59 |
+
"lstrip": false,
|
60 |
+
"normalized": true,
|
61 |
+
"rstrip": false,
|
62 |
+
"single_word": false
|
63 |
+
}
|
64 |
+
}
|