File size: 1,610 Bytes
2b6e7e3 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 |
Experimental 20B instruction tuned model based on gpt-neox-20b.
```
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch
name = "jordiclive/instruction-tuned-gpt-neox-20b"
model = AutoModelForCausalLM.from_pretrained(name, device_map=chip_map, torch_dtype=torch.float16)# load_in_8bit=True )
tokenizer = AutoTokenizer.from_pretrained(name)
def generate_from_model(model, tokenizer):
encoded_input = tokenizer(text, return_tensors='pt')
output_sequences = model.generate(
input_ids=encoded_input['input_ids'].cuda(0),
do_sample=True,
max_new_tokens=35,
num_return_sequences=1,
top_p=0.95,
temperature=0.5,
penalty_alpha=0.6,
top_k=4,
output_scores=True,
return_dict_in_generate=True,
repetition_penalty=1.03,
eos_token_id=0,
use_cache=True
)
gen_sequences = output_sequences.sequences[:, encoded_input['input_ids'].shape[-1]:]
for sequence in gen_sequences:
new_line=tokenizer.decode(sequence, skip_special_tokens=True)
print(new_line)
text = "User: Will big tech A.I be adulterated with advertisement?\n\nOA:"
generate_from_model(model,tokenizer)
```
|