add easy inference

README.md

@@ -13,9 +13,6 @@ model-index:
   results: []
 ---
 
-<!-- This model card has been generated automatically according to the information the Trainer had access to. You
-should probably proofread and complete it, then remove this comment. -->
-
 # gemma-2b-dolly-qa
 
 This model is a fine-tuned version of [google/gemma-2b](https://huggingface.co/google/gemma-2b) on the generator dataset.

run.py

@@ -0,0 +1,32 @@
+import torch, sys
+from peft import PeftModel
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+import warnings
+warnings.filterwarnings("ignore")
+
+access_token = sys.argv[2]
+device = "xpu:0" if sys.argv[1] == "gpu" else "cpu:0"
+
+tokenizer = AutoTokenizer.from_pretrained("./tokenizer/")
+tokenizer.pad_token = tokenizer.eos_token
+tokenizer.padding_side = "right"
+
+base_model = AutoModelForCausalLM.from_pretrained(
+    "google/gemma-2b",
+    token=access_token,
+    low_cpu_mem_usage=True,
+    return_dict=True,
+    torch_dtype=torch.bfloat16,
+)
+
+model = PeftModel.from_pretrained(base_model, "adapter_model")
+model = model.to(device)
+
+print("Prompt:", " ".join(sys.argv[3:]))
+
+inputs = tokenizer(" ".join(sys.argv[3:]), return_tensors="pt").to(device)
+outputs = model.generate(**inputs, max_new_tokens=200, 
+                            do_sample=False, top_k=100,temperature=0.1, 
+                            eos_token_id=tokenizer.eos_token_id)
+print(tokenizer.decode(outputs[0], skip_special_tokens=True))