Default template and configuration for local run with GPU
#33
by
brunoedcf
- opened
I have a template like this one:
- Is it possible to run mistral-7b using a similar template?
- What pytorch + cuda do I need to install?
- What dependencies are necessary?
from transformers import pipeline, AutoModelForCausalLM, AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained( )
pipeline = pipeline(
"text-generation",
model= ,
torch_dtype=torch.float16,
device_map="auto",
eos_token_id=tokenizer.eos_token_id,
pad_token_id=tokenizer.eos_token_id,
)
prompt = "prompt"
sequences = []
sequences.append(
pipeline(
prompt,
top_k=10,
max_new_tokens=150,
num_return_sequences=1,
)
)
for sequence in sequences:
generated_text = sequence[0]['generated_text']
print(generated_text)