Init app

app.py

@@ -10,10 +10,12 @@ app = FastAPI()
 base_model_name = "akjindal53244/Llama-3.1-Storm-8B"
 peft_model_id = "LlamaFactoryAI/cv-job-description-matching"
 
-base_model = AutoModelForCausalLM.from_pretrained(base_model_name)
+base_model = AutoModelForCausalLM.from_pretrained(base_model_name, torch_dtype=torch.float16)
+model = PeftModel.from_pretrained(base_model, peft_model_id, torch_dtype=torch.float16)
 tokenizer = AutoTokenizer.from_pretrained(base_model_name)
 config = PeftConfig.from_pretrained(peft_model_id)
-model = PeftModel.from_pretrained(base_model, peft_model_id)
+
+
 
 # Define request model
 class AnalysisRequest(BaseModel):
@@ -45,7 +47,7 @@ async def analyze(request: AnalysisRequest):
 
         # Tokenize and generate response
         inputs = tokenizer(input_text, return_tensors="pt")
-        outputs = model.generate(**inputs, max_new_tokens=128)
+        outputs = model.generate(**inputs, max_new_tokens=64)
         generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
 
         return {"analysis": generated_text}