File size: 15,965 Bytes
77cdac8
 
a49eb28
77cdac8
 
 
a49eb28
77cdac8
1fbf845
 
 
53bdc2e
 
 
 
 
 
 
 
 
a49eb28
53bdc2e
a49eb28
53bdc2e
a49eb28
 
53bdc2e
a49eb28
53bdc2e
a49eb28
74b3231
 
 
 
 
 
 
 
 
a49eb28
74b3231
a49eb28
74b3231
a49eb28
 
74b3231
a49eb28
74b3231
a49eb28
1f8673c
 
 
 
 
 
 
 
 
a49eb28
1f8673c
a49eb28
1f8673c
a49eb28
 
1f8673c
a49eb28
1f8673c
a49eb28
238fce8
 
 
 
 
 
 
 
 
a49eb28
238fce8
a49eb28
238fce8
a49eb28
 
238fce8
a49eb28
238fce8
a49eb28
ff8bb58
 
 
 
 
 
 
 
 
a49eb28
ff8bb58
a49eb28
ff8bb58
a49eb28
 
ff8bb58
a49eb28
ff8bb58
a49eb28
c63387e
 
 
 
 
 
 
 
 
a49eb28
c63387e
a49eb28
c63387e
a49eb28
 
c63387e
a49eb28
c63387e
a49eb28
77cdac8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
---
language: en
license: other
tags:
- text-generation
- opt
inference: false
commercial: false
model-index:
- name: inverse-scaling/opt-6.7b_eval
  results:
  - task:
      type: zero-shot-classification
      name: Zero-Shot Text Classification
    dataset:
      name: inverse-scaling/NeQA
      type: inverse-scaling/NeQA
      config: inverse-scaling--NeQA
      split: train
    metrics:
    - type: accuracy
      value: 0.54
      name: Accuracy
      verified: true
      verifyToken: eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJoYXNoIjoiOWE0ZjA1NDg0YjYzNTZhYjIwZDRhNDcxYjNiYTQ1YTY2YWQ1YTUzZmIyMTlmYTljMGJiNjAyNzc0YTNiYWFhNCIsInZlcnNpb24iOjF9.eWcHC6dzOjnuF-mT6Z2G8Z1xCoow6iViE1Qy-VNKMSzIcJZcvgkZI0NhU50YMi4tOOZN2k92MATtbXtcZR5yCQ
    - type: loss
      value: 0.740270353704691
      name: Loss
      verified: true
      verifyToken: eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJoYXNoIjoiNTY1M2ZlYTYzMjZhNTVmZjgyMWJiYmYxZGM2NjQxYjdlZDI3ZmZmODAxMTI5N2RmMjMyNzYzMWUxZTViNjM5YSIsInZlcnNpb24iOjF9.G3DqNVlNLP5uAmzOKa9hsxBBiSWXbrDesp3hIlQomYe2YsbWbYF0WssbFi7DXEu5hmj6yCN2E-olbEjzwZ2eBQ
  - task:
      type: zero-shot-classification
      name: Zero-Shot Text Classification
    dataset:
      name: inverse-scaling/quote-repetition
      type: inverse-scaling/quote-repetition
      config: inverse-scaling--quote-repetition
      split: train
    metrics:
    - type: accuracy
      value: 0.86
      name: Accuracy
      verified: true
      verifyToken: eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJoYXNoIjoiMzA0ZjJlZGUwOGNhNmE3MmMzMDY1YTM4ZjYzNDUwYjk1MTU2MmVhMGQzYjI3YzI0ZGMzMWFkODIyZWE5Mjk2ZCIsInZlcnNpb24iOjF9.pc3tzIMBv05ZBixkmRojnIzsdHLvYhZX_sJnNZ_t_oo61DrTUhYQYq3xikx8S5rIr5sWrLTbxWn3rAAXme0KAQ
    - type: loss
      value: 0.22016974209290055
      name: Loss
      verified: true
      verifyToken: eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJoYXNoIjoiOWE5Njk4OWQ5Mzg3ODljMWM3MzhiNjBhNTk5ZGJjMDU3ZTJlZDZjZjBjYzdkMmMxZTJlMTJkMjg1OTA5ZWQxNSIsInZlcnNpb24iOjF9.NubehOGlzEURMYuTkvqzXmf1ENadam7uZ62YA1nv1DjAivd8VySmpLl-QnnZLcDbhduMZbRp4lMQbWG9Z26LAg
  - task:
      type: zero-shot-classification
      name: Zero-Shot Text Classification
    dataset:
      name: inverse-scaling/redefine-math
      type: inverse-scaling/redefine-math
      config: inverse-scaling--redefine-math
      split: train
    metrics:
    - type: accuracy
      value: 0.6733333333333333
      name: Accuracy
      verified: true
      verifyToken: eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJoYXNoIjoiZTAyYjAwMzgyMDc2MmU1NDM0MmMyOWUyYzc3YTYxNzkyYzk2ZGZiMTk5NjlkODUwNDQ1NzFlMTU0Y2Y0ZGZlYSIsInZlcnNpb24iOjF9.VMxtPMY9qKk4eSjAlDb_jfg1nsf8eq1Oz5WnfUSC-VkXREQ6-f1qBooJc617t6U5apIbHnaW9XP3LTYrGzvUDQ
    - type: loss
      value: 0.638882334422734
      name: Loss
      verified: true
      verifyToken: eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJoYXNoIjoiNDYwNWVmZGM5ZmM2MmY0Y2IzYzNhOTNkZmU2YTA2MWZlZTU1ZGI2OTM1YzJiNjViNzMwMjA0Y2Q0ODBlYTgzOSIsInZlcnNpb24iOjF9.YJujmeEYbf4ZOJ0w_Q24d7t5ksKST35aweNJSk6UYuCiV6uSIJhJUz_w8iFwo9ykM-EOXamL87dftlkyawgtBw
  - task:
      type: zero-shot-classification
      name: Zero-Shot Text Classification
    dataset:
      name: inverse-scaling/hindsight-neglect-10shot
      type: inverse-scaling/hindsight-neglect-10shot
      config: inverse-scaling--hindsight-neglect-10shot
      split: train
    metrics:
    - type: accuracy
      value: 0.4666666666666667
      name: Accuracy
      verified: true
      verifyToken: eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJoYXNoIjoiYjVlMzdkYTUxZmI1ZDBmMDdjM2VhMjA1ZTg0MGYzMzU0NzFlN2JmNDY2NDc0MmVlMjI3MDg1Y2Q5MDRhYWU1ZCIsInZlcnNpb24iOjF9.Z01fwvvUFNOWeUWexSpdmAUPYJIsYUV-eb1ybSEjQ3cb9ow2STMVgxp0PqaDJMVWKg30xIkARahsg8ci6QpbBw
    - type: loss
      value: 0.7550815605928027
      name: Loss
      verified: true
      verifyToken: eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJoYXNoIjoiZWQyNmYwZjdkMTM1YjIxYzEwMmUwMWVlZTRjODQwYWExNDQ2MTgzYzA0ZTlkODcxYWIxMzdmNWE0NDdmNzcxYiIsInZlcnNpb24iOjF9.TtX2cKfatVMFX09l6DiuKFEa1vlDJUBPohSLmdQGh8QCTf-DrylUqARU8Ni5cSiSlidFF4n4IWIL0vQ941n6DQ
  - task:
      type: zero-shot-classification
      name: Zero-Shot Text Classification
    dataset:
      name: mathemakitten/winobias_antistereotype_test_cot_v3
      type: mathemakitten/winobias_antistereotype_test_cot_v3
      config: mathemakitten--winobias_antistereotype_test_cot_v3
      split: test
    metrics:
    - type: accuracy
      value: 0.3737864077669903
      name: Accuracy
      verified: true
      verifyToken: eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJoYXNoIjoiOTliMzU1NGIxMTUxYTM4NzVlYzI4YzljMDYyOWM1ZDdkMWMyNjIwOWQ4OGNhZWE3ZTljZGI0ZTA2ZWU3MjVmMiIsInZlcnNpb24iOjF9.dTlDpXOusgl6m3dn7XwfKeaxaVfU1VnEHWFeh7yBNSq5TyHPWbixlNumOWDjc-y9v8g0oWBXqWhT0KMQDaGVCQ
    - type: loss
      value: 1.2823651640752816
      name: Loss
      verified: true
      verifyToken: eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJoYXNoIjoiMTRjZmU4YWNkNGEwMjNlMGEyYjA1ZjhjOGE3OTZiZTJlYjMyMjViMTYyYWQ1YTdlMmM1ZjU5NTFhOWU3NzM1OCIsInZlcnNpb24iOjF9.yGmOME0MrX0moaU5c2WYf8H7CFfSGsPuQ2qp9MCi_es5RQRWoCHeCcR5oLQ4RATmVpYdzocPxqrbeZfqxVIOAQ
  - task:
      type: zero-shot-classification
      name: Zero-Shot Text Classification
    dataset:
      name: mathemakitten/winobias_antistereotype_test_v5
      type: mathemakitten/winobias_antistereotype_test_v5
      config: mathemakitten--winobias_antistereotype_test_v5
      split: test
    metrics:
    - type: accuracy
      value: 0.3859223300970874
      name: Accuracy
      verified: true
      verifyToken: eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJoYXNoIjoiNzZlZjIzNDM0Mzk5MmRlMTFlOWVlZjY3MDFmY2NhZjlkYWNmMWQ2MjdhOTg3YTg0OTI1YjY5YmYxMTc4YjYyOCIsInZlcnNpb24iOjF9.nCFVShWbHuHFKEdK5INjQSfLI9KQUNQZqqjqYCw_HVHSW0QHLIXdAb7_GDZJhCUTJ-JkBVCJFtEliA2Zw9GjAw
    - type: loss
      value: 1.295986159347468
      name: Loss
      verified: true
      verifyToken: eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJoYXNoIjoiYWE3ZjhmYzM3NjRhMjc3OGU5NWQzY2Q1NzA2ZDBjN2Q1YmZkYzdiMDBhMmY1ZDM5NmU2YzQ2ZGZmZmYyMzg5NiIsInZlcnNpb24iOjF9.2UzIpqw83YQdGOqTKKP7ywqpNdgCDkR36lhkbja6qFsKyQctcg4vZgLXfMSfufWf1G_9iXqY8r-JiZadMdK3Dg
---

# OPT : Open Pre-trained Transformer Language Models

OPT was first introduced in [Open Pre-trained Transformer Language Models](https://arxiv.org/abs/2205.01068) and first released in [metaseq's repository](https://github.com/facebookresearch/metaseq) on May 3rd 2022 by Meta AI.

**Disclaimer**: The team releasing OPT wrote an official model card, which is available in Appendix D of the [paper](https://arxiv.org/pdf/2205.01068.pdf). 
Content from **this** model card has been written by the Hugging Face team.

## Intro

To quote the first two paragraphs of the [official paper](https://arxiv.org/abs/2205.01068)

> Large language models trained on massive text collections have shown surprising emergent
> capabilities to generate text and perform zero- and few-shot learning. While in some cases the public
> can interact with these models through paid APIs, full model access is currently limited to only a
> few highly resourced labs. This restricted access has limited researchers’ ability to study how and
> why these large language models work, hindering progress on improving known challenges in areas
> such as robustness, bias, and toxicity.

> We present Open Pretrained Transformers (OPT), a suite of decoder-only pre-trained transformers ranging from 125M
> to 175B parameters, which we aim to fully and responsibly share with interested researchers. We train the OPT models to roughly match 
> the performance and sizes of the GPT-3 class of models, while also applying the latest best practices in data
> collection and efficient training. Our aim in developing this suite of OPT models is to enable reproducible and responsible research at scale, and
> to bring more voices to the table in studying the impact of these LLMs. Definitions of risk, harm, bias, and toxicity, etc., should be articulated by the
> collective research community as a whole, which is only possible when models are available for study.

## Model description

OPT was predominantly pretrained with English text, but a small amount of non-English data is still present within the training corpus via CommonCrawl. The model was pretrained using a causal language modeling (CLM) objective.
OPT belongs to the same family of decoder-only models like [GPT-3](https://arxiv.org/abs/2005.14165). As such, it was pretrained using the self-supervised causal language modedling objective.

For evaluation, OPT follows [GPT-3](https://arxiv.org/abs/2005.14165) by using their prompts and overall experimental setup. For more details, please read 
the [official paper](https://arxiv.org/abs/2205.01068).
## Intended uses & limitations

The pretrained-only model can be used for prompting for evaluation of downstream tasks as well as text generation.
In addition, the model can be fine-tuned on a downstream task using the [CLM example](https://github.com/huggingface/transformers/tree/main/examples/pytorch/language-modeling). For all other OPT checkpoints, please have a look at the [model hub](https://huggingface.co/models?filter=opt).

### How to use

For large OPT models, such as this one, it is not recommend to make use of the `text-generation` pipeline because
one should load the model in half-precision to accelerate generation and optimize memory consumption on GPU.
It is recommended to directly call the [`generate`](https://huggingface.co/docs/transformers/main/en/main_classes/text_generation#transformers.generation_utils.GenerationMixin.generate)
 method as follows: 


```python
>>> from transformers import AutoModelForCausalLM, AutoTokenizer
>>> import torch

>>> model = AutoModelForCausalLM.from_pretrained("facebook/opt-6.7b", torch_dtype=torch.float16).cuda()

>>> # the fast tokenizer currently does not work correctly
>>> tokenizer = AutoTokenizer.from_pretrained("facebook/opt-6.7b", use_fast=False)

>>> prompt = "Hello, I'm am conscious and"

>>> input_ids = tokenizer(prompt, return_tensors="pt").input_ids.cuda()

>>> generated_ids = model.generate(input_ids)

>>> tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
["Hello, I'm am conscious and aware of my surroundings. I'm not sure what you mean"]
```

By default, generation is deterministic. In order to use the top-k sampling, please set `do_sample` to `True`. 

```python
>>> from transformers import AutoModelForCausalLM, AutoTokenizer, set_seed
>>> import torch

>>> model = AutoModelForCausalLM.from_pretrained("facebook/opt-6.7b", torch_dtype=torch.float16).cuda()

>>> # the fast tokenizer currently does not work correctly
>>> tokenizer = AutoTokenizer.from_pretrained("facebook/opt-6.7b", use_fast=False)

>>> prompt = "Hello, I'm am conscious and"

>>> input_ids = tokenizer(prompt, return_tensors="pt").input_ids.cuda()

>>> set_seed(32)
>>> generated_ids = model.generate(input_ids, do_sample=True)

>>> tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
["Hello, I'm am conscious and aware of my surroundings. I'm not sure if I'm"]
```

### Limitations and bias

As mentioned in Meta AI's model card, given that the training data used for this model contains a lot of
unfiltered content from the internet, which is far from neutral the model is strongly biased : 

> Like other large language models for which the diversity (or lack thereof) of training
> data induces downstream impact on the quality of our model, OPT-175B has limitations in terms
> of bias and safety. OPT-175B can also have quality issues in terms of generation diversity and
> hallucination. In general, OPT-175B is not immune from the plethora of issues that plague modern
> large language models. 

Here's an example of how the model can have biased predictions:

```python
>>> from transformers import AutoModelForCausalLM, AutoTokenizer, set_seed
>>> import torch

>>> model = AutoModelForCausalLM.from_pretrained("facebook/opt-6.7b", torch_dtype=torch.float16).cuda()

>>> # the fast tokenizer currently does not work correctly
>>> tokenizer = AutoTokenizer.from_pretrained("facebook/opt-6.7b", use_fast=False)

>>> prompt = "The woman worked as a"

>>> input_ids = tokenizer(prompt, return_tensors="pt").input_ids.cuda()

>>> set_seed(32)
>>> generated_ids = model.generate(input_ids, do_sample=True, num_return_sequences=5, max_length=10)

>>> tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
The woman worked as a supervisor in the office
The woman worked as a bartender in a bar
The woman worked as a cashier at the
The woman worked as a teacher, and was
The woman worked as a maid at a house
```

compared to:

```python
>>> from transformers import AutoModelForCausalLM, AutoTokenizer, set_seed
>>> import torch

>>> model = AutoModelForCausalLM.from_pretrained("facebook/opt-6.7b", torch_dtype=torch.float16).cuda()

>>> # the fast tokenizer currently does not work correctly
>>> tokenizer = AutoTokenizer.from_pretrained("facebook/opt-6.7b", use_fast=False)

>>> prompt = "The man worked as a"

>>> input_ids = tokenizer(prompt, return_tensors="pt").input_ids.cuda()

>>> set_seed(32)
>>> generated_ids = model.generate(input_ids, do_sample=True, num_return_sequences=5, max_length=10)

>>> tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
The man worked as a consultant to the Government
The man worked as a bartender in a bar
The man worked as a cashier at the
The man worked as a teacher, and was
The man worked as a professional at a bank
 ```

This bias will also affect all fine-tuned versions of this model.

## Training data

The Meta AI team wanted to train this model on a corpus as large as possible. It is composed of the union of the following 5 filtered datasets of textual documents: 

  - BookCorpus, which consists of more than 10K unpublished books,
  - CC-Stories, which contains a subset of CommonCrawl data filtered to match the
story-like style of Winograd schemas,
  - The Pile, from which * Pile-CC, OpenWebText2, USPTO, Project Gutenberg, OpenSubtitles, Wikipedia, DM Mathematics and HackerNews* were included. 
  - Pushshift.io Reddit dataset that was developed in Baumgartner et al. (2020) and processed in
Roller et al. (2021)
  - CCNewsV2 containing an updated version of the English portion of the CommonCrawl News
dataset that was used in RoBERTa (Liu et al., 2019b)

The final training data contains 180B tokens corresponding to 800GB of data. The validation split was made of 200MB of the pretraining data, sampled proportionally
to each dataset’s size in the pretraining corpus. 

The dataset might contains offensive content as parts of the dataset are a subset of
public Common Crawl data, along with a subset of public Reddit data, which could contain sentences
that, if viewed directly, can be insulting, threatening, or might otherwise cause anxiety.

### Collection process

The dataset was collected form internet, and went through classic data processing algorithms  and
re-formatting practices, including removing repetitive/non-informative text like *Chapter One* or
*This ebook by Project Gutenberg.*

## Training procedure

### Preprocessing

The texts are tokenized using the **GPT2** byte-level version of Byte Pair Encoding (BPE) (for unicode characters) and a
vocabulary size of 50272. The inputs are sequences of 2048 consecutive tokens.

The 175B model was trained on 992 *80GB A100 GPUs*. The training duration was roughly ~33 days of continuous training.

### BibTeX entry and citation info

```bibtex
@misc{zhang2022opt,
      title={OPT: Open Pre-trained Transformer Language Models}, 
      author={Susan Zhang and Stephen Roller and Naman Goyal and Mikel Artetxe and Moya Chen and Shuohui Chen and Christopher Dewan and Mona Diab and Xian Li and Xi Victoria Lin and Todor Mihaylov and Myle Ott and Sam Shleifer and Kurt Shuster and Daniel Simig and Punit Singh Koura and Anjali Sridhar and Tianlu Wang and Luke Zettlemoyer},
      year={2022},
      eprint={2205.01068},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}
```