Update README.md

README.md

@@ -36,33 +36,53 @@ This model is a fine-tuned version of [SGaleshchuk/Llama-2-13b-hf_uk_rank-32_ft]
 
 ## Intended uses & limitations
 ```python
+# tested with colab+A100 GPU
+!pip install -q -U peft transformers==4.30
+!pip install flash-attn --no-build-isolation
+!pip install einops bitsandbytes accelerate
 # unpatch flash attention
+import torch
 from peft import AutoPeftModelForCausalLM
 from transformers import AutoTokenizer
 
+model_id = "SGaleshchuk/Llama-2-13b-summarization_uk_dpo"
+
 # load base LLM model and tokenizer
 model = AutoPeftModelForCausalLM.from_pretrained(
-  "SGaleshchuk/Llama-2-13b-summarization_uk_dpo",
+  model_id,
   low_cpu_mem_usage=True,
   torch_dtype=torch.float16,
   load_in_4bit=True)
 
-tokenizer = AutoTokenizer.from_pretrained(peft_model_id)
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+
+def prepare_instruction(text):
+
 
-for instruct, summary in zip(val_instructions, tqdm(summaries)):
-    input_ids = tokenizer(
-       instruct, return_tensors="pt", truncation=True).input_ids.cuda()
+    prompt = """The article to summarize in maximum 100 words:{text}. Summary:""" # adapt to your needs
+
+    return prompt.format(
+        text=text,
+    )
+def summarization(text):
+    instruction = prepare_instruction(text)
+    input_ids = tokenizer(instruction, return_tensors="pt", truncation=True).input_ids.cuda()
     with torch.inference_mode():
-        outputs = model.generate(
-                input_ids=input_ids,
-                max_new_tokens=128,
-                do_sample=True,
-                top_p=0.9,
-                temperature=1e-2,
+      outputs = model.generate(
+              input_ids=input_ids,
+              max_new_tokens=128,
+              do_sample=True,
+              top_p=0.9,
+              temperature=1e-2,
             )
-        result = tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True)[0]
-        result = result[len(instruct) :]
-        print(result)
+      result = tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True)[0]
+      result = result[len(instruction) :]
+      print(result)
+      return result
+
+text = """your text here to summarize"
+result = summarization(text)
+
 ```
 
 ## Training procedure