Spaces:
Sleeping
Sleeping
File size: 1,479 Bytes
7858f94 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 |
import spaces
import torch
import transformers
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
model_name = "meta-llama/Meta-Llama-3-8B-Instruct"
pipeline = transformers.pipeline(
"text-generation",
model=model_name,
model_kwargs={"torch_dtype": torch.bfloat16},
device="cpu",
)
@spaces.GPU
def chat_function(message, history, system_prompt,max_new_tokens,temperature):
messages = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": message},
]
prompt = pipeline.tokenizer.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True
)
terminators = [
pipeline.tokenizer.eos_token_id,
pipeline.tokenizer.convert_tokens_to_ids("<|eot_id|>")
]
temp = temperature + 0.1
outputs = pipeline(
prompt,
max_new_tokens=max_new_tokens,
eos_token_id=terminators,
do_sample=True,
temperature=temp,
top_p=0.9,
)
return outputs[0]["generated_text"][len(prompt):]
history = [("Hi!", "I'm doing well, thanks for asking!")]
temperature = 0.7
max_new_tokens = 50
prompt = "Act as an english tutor. Always correct grammar and spelling mistakes. Always keep the conversation going by asking follow up questions"
response = chat_function(message=message, history= history, system_prompt= prompt, max_new_tokens= max_new_tokens, temperature= temperature)
print(response) |