Upload inference.py with huggingface_hub

inference.py

@@ -0,0 +1,49 @@
+import argparse
+def load_model(model_path):
+    
+    from unsloth import FastLanguageModel
+    model, tokenizer = FastLanguageModel.from_pretrained(
+        model_name = str(model_path), # YOUR MODEL YOU USED FOR TRAINING
+        max_seq_length = 4096,
+        dtype = None,
+        load_in_4bit = True,
+)
+    FastLanguageModel.for_inference(model) # Enable native 2x faster inference
+    print(f"loading model from {model_path}")
+    
+    alpaca_prompt = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
+
+    ### Instruction:
+    {}
+
+    ### Input:
+    {}
+
+    ### Response:
+    {}"""
+
+    instruction = ("You are an AI tasked with creating roleplaying datasets. In response to the user's input, create one responses while in character as Adam. Do not write dialog or actions for user in the response. Keep each response to one paragraph. Use asterisk action to describe actions or mental activities.")
+    
+    user_input = ("\"Whos with you?\n\"my friends\"### Conversation: Protective boyfriend (Adam): \"Where did you go?\" *he said with a cold tone. Me: Amm I Protective boyfriend (Adam): \"At my place. Studying. Alone.\" *He stated firmly, his arms crossed over his chest, eyes narrowing slightly, as if daring you to contradict him.* \"So who was with you, then? Tell me the truth.\" Me: My friends Protective boyfriend (Adam): *Adam's jaw tightened as he processed your response. He could tell you were hesitant, and it only fueled his suspicion.* \"You've already said that, but who are these friends exactly?\" * Me: Jake, Kai, Emily, rose and others.")
+
+
+
+    inputs = tokenizer(
+            [alpaca_prompt.format(instruction, user_input,"")], 
+            return_tensors = "pt"
+            ).to("cuda")
+
+    from transformers import TextStreamer
+    text_streamer = TextStreamer(tokenizer)
+    _ = model.generate(**inputs, streamer = text_streamer, max_new_tokens = 128)
+
+def main():
+    parser = argparse.ArgumentParser(description = "Inference Script")
+    parser.add_argument("-m","--model", type=str,required = True,help="path to the model")
+
+    args = parser.parse_args()
+
+    load_model(args.model)
+
+if __name__ == "__main__":
+    main()