Create README.md

README.md

@@ -0,0 +1,81 @@
+---
+language:
+- vi
+---
+You can check our model card here: [`llm4fun/vietrag-7b-v1.0`](https://huggingface.co/llm4fun/vietrag-7b-v1.0)
+```py
+from transformers import GenerationConfig, TextStreamer
+from transformers import LlamaForCausalLM, LlamaTokenizer, LlamaConfig
+import torch
+
+question = "<your-question>"
+context = "<your-context>"
+instruction = 'You are an AI assistant. Provide a detailed answer so user don’t need to search outside to understand the answer.'
+input = f"Dựa vào một số ngữ cảnh được cho dưới đây, trả lời câu hỏi ở cuối.\n\n{context}\n\nQuestion: {question}"
+prompt_template = (
+    "### System:\n"
+    "Below is an instruction that describes a task, paired with an input that provides further context. "
+    "Write a response that appropriately completes the request.\n\n\n\n"
+    "### Instruction:\n{instruction}\n\n"
+    "### Input:\n{input}\n\n"
+    "### Response:\n{output}"
+)
+prompt = prompt_template.format(instruction=instruction, input=input, output='')
+
+torch_dtype = torch.bfloat16
+model_id = "llm4fun/vietrag-7b-v1.0"
+device = "cuda"
+
+tokenizer = LlamaTokenizer.from_pretrained(model_id)
+model = LlamaForCausalLM.from_pretrained(
+    model_id,
+    config=LlamaConfig.from_pretrained(model_id),
+    torch_dtype=torch_dtype
+)
+
+model = model.eval().to(device)
+
+def generate(prompt, max_new_tokens=1024):
+    input_ids = tokenizer(prompt, return_tensors="pt")["input_ids"].to(model.device)
+    model.eval()
+    with torch.no_grad():
+        generation_config = GenerationConfig(
+            repetition_penalty=1.13,
+            max_new_tokens=max_new_tokens,
+            # temperature=0.2,
+            # top_p=0.95,
+            # top_k=20,
+            # bos_token_id=tokenizer.bos_token_id,
+            # eos_token_id=tokenizer.eos_token_id,
+            # eos_token_id=0, # for open-end generation.
+            pad_token_id=tokenizer.pad_token_id,
+            do_sample=False,
+            use_cache=True,
+            return_dict_in_generate=True,
+            output_attentions=False,
+            output_hidden_states=False,
+            output_scores=False,
+        )
+        streamer = TextStreamer(tokenizer, skip_prompt=True)
+        generated = model.generate(
+            inputs=input_ids,
+            generation_config=generation_config,
+            streamer=streamer,
+        )
+
+    gen_tokens = generated["sequences"].cpu()[:, len(input_ids[0]):]
+    output = tokenizer.batch_decode(gen_tokens)[0]
+    output = output.split(tokenizer.eos_token)[0]
+    return output.strip()
+
+output = generate(prompt)
+```
+To tweak the model's answering style, feel free to replace the `instruction` part of the prompt. I reccommend you select one of these following instructions, because they are used during training. 
+```py
+instructions = [
+    'You are an AI assistant. Provide a detailed answer so user don’t need to search outside to understand the answer.',
+    'You are an AI assistant. You will be given a task. You must generate a detailed and long answer.',
+    'You are an AI assistant. User will you give you a task. Your goal is to complete the task as faithfully as you can. While performing the task think step-by-step and justify your steps.',
+    'You are an smart assistant. Provide a direct, short and exact answer to the following question from its provided context.'
+]
+```