File size: 3,357 Bytes
b6f999e 12a10e9 b6f999e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 |
---
language:
- vi
---
# Usage
You can check our model card here: [`llm4fun/vietrag-7b-v1.0`](https://huggingface.co/llm4fun/vietrag-7b-v1.0)
```py
from transformers import GenerationConfig, TextStreamer
from transformers import LlamaForCausalLM, LlamaTokenizer, LlamaConfig
import torch
question = "<your-question>"
context = "<your-context>"
instruction = 'You are an AI assistant. Provide a detailed answer so user don’t need to search outside to understand the answer.'
input = f"Dựa vào một số ngữ cảnh được cho dưới đây, trả lời câu hỏi ở cuối.\n\n{context}\n\nQuestion: {question}"
prompt_template = (
"### System:\n"
"Below is an instruction that describes a task, paired with an input that provides further context. "
"Write a response that appropriately completes the request.\n\n\n\n"
"### Instruction:\n{instruction}\n\n"
"### Input:\n{input}\n\n"
"### Response:\n{output}"
)
prompt = prompt_template.format(instruction=instruction, input=input, output='')
torch_dtype = torch.bfloat16
model_id = "llm4fun/vietrag-7b-v1.0"
device = "cuda"
tokenizer = LlamaTokenizer.from_pretrained(model_id)
model = LlamaForCausalLM.from_pretrained(
model_id,
config=LlamaConfig.from_pretrained(model_id),
torch_dtype=torch_dtype
)
model = model.eval().to(device)
def generate(prompt, max_new_tokens=1024):
input_ids = tokenizer(prompt, return_tensors="pt")["input_ids"].to(model.device)
model.eval()
with torch.no_grad():
generation_config = GenerationConfig(
repetition_penalty=1.13,
max_new_tokens=max_new_tokens,
# temperature=0.2,
# top_p=0.95,
# top_k=20,
# bos_token_id=tokenizer.bos_token_id,
# eos_token_id=tokenizer.eos_token_id,
# eos_token_id=0, # for open-end generation.
pad_token_id=tokenizer.pad_token_id,
do_sample=False,
use_cache=True,
return_dict_in_generate=True,
output_attentions=False,
output_hidden_states=False,
output_scores=False,
)
streamer = TextStreamer(tokenizer, skip_prompt=True)
generated = model.generate(
inputs=input_ids,
generation_config=generation_config,
streamer=streamer,
)
gen_tokens = generated["sequences"].cpu()[:, len(input_ids[0]):]
output = tokenizer.batch_decode(gen_tokens)[0]
output = output.split(tokenizer.eos_token)[0]
return output.strip()
output = generate(prompt)
```
To tweak the model's answering style, feel free to replace the `instruction` part of the prompt. I reccommend you select one of these following instructions, because they are used during training.
```py
instructions = [
'You are an AI assistant. Provide a detailed answer so user don’t need to search outside to understand the answer.',
'You are an AI assistant. You will be given a task. You must generate a detailed and long answer.',
'You are an AI assistant. User will you give you a task. Your goal is to complete the task as faithfully as you can. While performing the task think step-by-step and justify your steps.',
'You are an smart assistant. Provide a direct, short and exact answer to the following question from its provided context.'
]
``` |