working version

app.py

@@ -1,29 +1,134 @@
+import logging
 import gradio as gr
-from transformers import AutoModelForCausalLM, AutoTokenizer
+import wandb
+import transformers
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+import peft
+from peft import PeftModel
+import torch
 
-# Assuming you've uploaded your model to Hugging Face with the username 'lehmanc25'
-tokenizer = AutoTokenizer.from_pretrained("lehmanc25/my-phi2-model")
-model = AutoModelForCausalLM.from_pretrained("lehmanc25/my-phi2-model")
+wandb.login()
+wandb.init(project='journal-finetune', entity='benbankston2')
 
-def chat_with_phi2(input_text):
-    encoded_input = tokenizer(input_text, return_tensors='pt')
-    output = model.generate(**encoded_input, max_length=50)
-    response = tokenizer.decode(output[0], skip_special_tokens=True)
+
+# Initialize logging
+logging.basicConfig(level=logging.INFO)
+
+base_model_id = "microsoft/phi-2"
+model = AutoModelForCausalLM.from_pretrained(
+    base_model_id,    
+    device_map="auto",
+    quantization_config=BitsAndBytesConfig(
+        load_in_4bit=True,
+        bnb_4bit_compute_dtype=torch.bfloat16,
+        bnb_4bit_quant_type="nf4",
+    ),
+    torch_dtype=torch.bfloat16,
+    # FA2 does not work yet
+    # attn_implementation="flash_attention_2",          
+)
+
+
+#model = pipeline("text-generation", model=model_name)
+model = PeftModel.from_pretrained(model, "phi2-journal-finetune/checkpoint-175")
+model.to("cuda")
+
+tokenizer = AutoTokenizer.from_pretrained(base_model_id, add_bos_token=True, trust_remote_code=True, use_fast=False)
+tokenizer.pad_token = tokenizer.eos_token
+
+def generate_text(prompt):
+    logging.info(f"Generating text for prompt: {prompt}")
+    model_input = tokenizer(prompt, return_tensors="pt").to("cuda")#100
+    #response = model(prompt, max_new_tokens=100, temperature=0.6, top_p=0.8, repetition_penalty=2.5, do_sample=True)
+    response = tokenizer.decode(model.generate(
+        **model_input, max_new_tokens=256, 
+        repetition_penalty=1.11)[0], 
+        temperature = 1,
+        eos_token_id=tokenizer.pad_token,
+        skip_special_tokens=True,
+        early_stopping = True,
+        )
+    #best_response = response[0]['generated_text']
+    logging.info(f"Generated text: {response}")
     return response
 
-# Using Gradio Blocks to build a more dynamic interface
-block = gr.Blocks()
-
-with block:
-    gr.Markdown("# W&L ChatGPT-like AI")
-    gr.Markdown("Ask any question about Washington and Lee University!")
-    input_text = gr.Textbox(label="Your question:", lines=2, placeholder="Ask something about W&L...")
-    output_text = gr.Text(label="AI Response:")
-    button = gr.Button("Ask")
-    button.click(
-        fn=chat_with_phi2, 
-        inputs=input_text, 
-        outputs=output_text
-    )
-
-block.launch()
+def message_and_history(input_text, history, feedback = None):
+    """Manage message history and generate responses."""#100
+    if history is None:
+        history = []
+    history2 = list(sum(history, ()))
+    history2.append(input_text)
+    input = ''.join(history2)
+    output = generate_text(input_text)#input)
+    history.append(("User", input_text))
+    history.append(("Fizz Bot", output))
+    return history, history
+
+def setup_interface():
+    with gr.Blocks(css='''
+        body { font-family: 'Arial', sans-serif; background: #f1f1f1; }
+        .container { max-width: 800px; margin: auto; padding: 20px; background-size: cover; background-repeat: no-repeat; border-radius: 8px; box-shadow: 0 4px 8px rgba(0,0,0,0.1); }
+        h1 { color: #003F87; text-align: center; margin-bottom: 20px; }
+        .gr-textbox { box-shadow: inset 0 2px 3px rgba(0,0,0,0.1); border-radius: 4px; border: 1px solid #7FB2E5; padding: 10px; width: auto; background-color: #fff; }
+        .gradio-chatbox { background-color: #f0f0f0; }
+        parameter-accordion .gr-accordion-title { font-weight: bold; font-size: 18px; } /* Custom CSS for accordion title */
+        .gradio-chatbox-message-user { background-color: #4A90E2; color: white; }
+        .gradio-chatbox-message-bot { background-color: #FFFFFF; color: black; }
+        .gradio-chatbox-message { border-radius: 10px; padding: 10px; margin-bottom: 8px; }
+        ''', theme ="Soft") as block:
+
+        gr.Markdown("""
+            <div style="background-image: url('https://my.wlu.edu/Images/communications/publications/graphic-identity/300-dpi-wordmark-blue.png'); 
+            background-size: contain; 
+            background-repeat: no-repeat; 
+            background-position: center; 
+            text-align: center; 
+            height: 100px; 
+            line-height: 100px; 
+            font-size: 36px; 
+            color: white; 
+            font-family: Arial, sans-serif;">
+            </div>
+        """)
+        gr.Markdown("<h1>Fizz Chatbot</h1>")
+        gr.Markdown("<h6><i>Disclaimer: some information may be inaccurate</i></h6>")
+        with gr.Accordion("Parameters", open=False, visible=True, elem_classes=["parameter-accordion"]) as parameter_row:
+            temperature = gr.Slider(
+                minimum = 0.0,
+                maximum = 1.0,
+                value = 0.7, 
+                step=0.1,
+                interactive = True,
+                label="Temperature"
+            )
+            top_p = gr.Slider(
+                minimum = 0.0,
+                maximum = 1.0,
+                value = 1.0, 
+                step=0.1,
+                interactive = True,
+                label="Top P"
+            )
+            max_new_tokens = gr.Slider(
+                minimum = 16,
+                maximum = 1028,
+                value = 128, 
+                step= 32,
+                interactive = True,
+                label="Max tokens"
+            )
+        chatbot = gr.Chatbot(label="W&L AI")
+        message = gr.Textbox(label="", placeholder="Ask me anything about W&L here...", elem_id="input_box")
+        submit = gr.Button("Submit Query", elem_classes="specific_button")
+        submit.click(
+            fn=message_and_history,
+            inputs=[message, gr.State()],
+            outputs=[chatbot, gr.State()]
+        )
+        gr.Row([chatbot])
+        gr.Row([message, submit])
+
+    return block
+
+app = setup_interface()
+app.launch(debug=True)