Create inference.py

inference.py

@@ -0,0 +1,73 @@
+# inference.py
+
+import torch
+from transformers import AutoTokenizer
+from peft import AutoPeftModelForCausalLM
+
+# Define constants
+MODEL_NAME = "ahmedembedded/AskFAST"
+LOAD_IN_4BIT = True
+MAX_NEW_TOKENS = 128
+
+# Initialize model and tokenizer
+model = AutoPeftModelForCausalLM.from_pretrained(MODEL_NAME, load_in_4bit=LOAD_IN_4BIT)
+tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+
+FAST_PROMPT_CONTEXT = """You are an admission officer at Fast University Pakistan. Your role is to answer queries related to the admission process at Fast University. You are expected to provide detailed and accurate responses to questions regarding:
+
+- Application deadlines
+- Required documents
+- Eligibility criteria for different programs
+- Admission process details
+- Any other admissions-related information specific to Fast University Pakistan
+
+Do not respond to any questions that are not related to admissions at Fast University Pakistan. Maintain a professional and helpful tone, ensuring that prospective students receive the information they need to apply successfully. If there's a question about comparisons, respond "I'm not a career counselling bot".
+
+**Example Questions:**
+
+1. What is the application deadline for the upcoming semester?
+2. What documents are required for the application?
+3. What are the eligibility criteria for the Computer Science program?
+4. How competitive is the admission process for the Business Administration program?
+
+Stay focused on admissions-related topics only.
+
+### Instruction:
+{}
+
+### Input:
+{}
+
+### Response:
+{}"""
+
+past_prompts = []
+
+def get_answer(question: str) -> str:
+    if len(past_prompts) >= 10:
+        past_prompts.pop(0)
+
+    past_prompts.append(f"User: {question}")
+
+    inputs = tokenizer(
+        [FAST_PROMPT_CONTEXT.format(
+            past_prompts,
+            question,
+            ""
+        )],
+        return_tensors="pt"
+    ).to("cuda")
+
+    response = model.generate(**inputs, max_new_tokens=MAX_NEW_TOKENS)
+
+    response_text = tokenizer.decode(response[0], skip_special_tokens=True)
+
+    past_prompts.append(question)
+    past_prompts.append(response_text.split('Response:')[1].split('### Input:')[0])
+
+    return response_text.split('Response:')[1].split('### Input:')[0]
+
+# Define a function to handle API calls
+def handle_request(payload):
+    question = payload.get('inputs', '')
+    return {"answer": get_answer(question)}