Create app.py

app.py

@@ -0,0 +1,67 @@
+from huggingface_hub import login
+import os
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+from peft import PeftModel
+import gradio as gr
+
+# Log in using the secret token
+login(os.environ["HF_TOKEN"])
+
+# Base model
+base_model = "mistralai/Mistral-7B-v0.3"
+
+# Your adapter model on HF
+adapter_model = "hin123123/theralingua-mistral-7b-word"
+
+# Quantization config for efficiency
+quantization_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_compute_dtype=torch.float16,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type="nf4"
+)
+
+# Load tokenizer
+tokenizer = AutoTokenizer.from_pretrained(base_model)
+
+# Load base model with low CPU memory usage
+model = AutoModelForCausalLM.from_pretrained(
+    base_model,
+    quantization_config=quantization_config,
+    device_map="auto",
+    low_cpu_mem_usage=True  # Streams to GPU if available, avoids full RAM load
+)
+
+# Apply LoRA adapter
+model = PeftModel.from_pretrained(model, adapter_model)
+
+def generate_text(input_text, max_new_tokens=100, temperature=0.7):
+    inputs = tokenizer(input_text, return_tensors="pt").to("cuda" if torch.cuda.is_available() else "cpu")
+    
+    with torch.inference_mode():
+        outputs = model.generate(
+            **inputs,
+            max_new_tokens=max_new_tokens,
+            temperature=temperature,
+            do_sample=True,
+            top_p=0.9
+        )
+    
+    generated = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    return generated
+
+demo = gr.Interface(
+    fn=generate_text,
+    inputs=[
+        gr.Textbox(label="Input Text", placeholder="Enter your prompt here..."),
+        gr.Slider(label="Max New Tokens", minimum=50, maximum=500, value=100, step=50),
+        gr.Slider(label="Temperature", minimum=0.1, maximum=1.5, value=0.7, step=0.1)
+    ],
+    outputs=gr.Textbox(label="Generated Output"),
+    title="Theralingua-Mistral-7B-Word Demo",
+    description="Enter text to generate output from the model."
+)
+
+# Launch the demo (Spaces handles sharing automatically)
+demo.launch()