Spaces:
Runtime error
Runtime error
| import logging | |
| import gradio as gr | |
| import wandb | |
| import transformers | |
| from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig | |
| import peft | |
| from peft import PeftModel | |
| import torch | |
| wandb.login(key = '926692a60ef1695538fcd1ae6e75482b0741a3d3') | |
| wandb.init(project='journal-finetune', entity='benbankston2') | |
| # Initialize logging | |
| logging.basicConfig(level=logging.INFO) | |
| base_model_id = "microsoft/phi-2" | |
| model = AutoModelForCausalLM.from_pretrained( | |
| base_model_id, | |
| device_map="auto", | |
| quantization_config=BitsAndBytesConfig( | |
| load_in_4bit=True, | |
| bnb_4bit_compute_dtype=torch.bfloat16, | |
| bnb_4bit_quant_type="nf4", | |
| ), | |
| torch_dtype=torch.bfloat16, | |
| # FA2 does not work yet | |
| # attn_implementation="flash_attention_2", | |
| ) | |
| #model = pipeline("text-generation", model=model_name) | |
| model = PeftModel.from_pretrained(model, "phi2-journal-finetune/checkpoint-175") | |
| model.to("cuda") | |
| tokenizer = AutoTokenizer.from_pretrained(base_model_id, add_bos_token=True, trust_remote_code=True, use_fast=False) | |
| tokenizer.pad_token = tokenizer.eos_token | |
| def generate_text(prompt): | |
| logging.info(f"Generating text for prompt: {prompt}") | |
| model_input = tokenizer(prompt, return_tensors="pt").to("cuda")#100 | |
| #response = model(prompt, max_new_tokens=100, temperature=0.6, top_p=0.8, repetition_penalty=2.5, do_sample=True) | |
| response = tokenizer.decode(model.generate( | |
| **model_input, max_new_tokens=256, | |
| repetition_penalty=1.11)[0], | |
| temperature = 1, | |
| eos_token_id=tokenizer.pad_token, | |
| skip_special_tokens=True, | |
| early_stopping = True, | |
| ) | |
| #best_response = response[0]['generated_text'] | |
| logging.info(f"Generated text: {response}") | |
| return response | |
| def message_and_history(input_text, history, feedback = None): | |
| """Manage message history and generate responses."""#100 | |
| if history is None: | |
| history = [] | |
| history2 = list(sum(history, ())) | |
| history2.append(input_text) | |
| input = ''.join(history2) | |
| output = generate_text(input_text)#input) | |
| history.append(("User", input_text)) | |
| history.append(("Fizz Bot", output)) | |
| return history, history | |
| def setup_interface(): | |
| with gr.Blocks(css=''' | |
| body { font-family: 'Arial', sans-serif; background: #f1f1f1; } | |
| .container { max-width: 800px; margin: auto; padding: 20px; background-size: cover; background-repeat: no-repeat; border-radius: 8px; box-shadow: 0 4px 8px rgba(0,0,0,0.1); } | |
| h1 { color: #003F87; text-align: center; margin-bottom: 20px; } | |
| .gr-textbox { box-shadow: inset 0 2px 3px rgba(0,0,0,0.1); border-radius: 4px; border: 1px solid #7FB2E5; padding: 10px; width: auto; background-color: #fff; } | |
| .gradio-chatbox { background-color: #f0f0f0; } | |
| parameter-accordion .gr-accordion-title { font-weight: bold; font-size: 18px; } /* Custom CSS for accordion title */ | |
| .gradio-chatbox-message-user { background-color: #4A90E2; color: white; } | |
| .gradio-chatbox-message-bot { background-color: #FFFFFF; color: black; } | |
| .gradio-chatbox-message { border-radius: 10px; padding: 10px; margin-bottom: 8px; } | |
| ''', theme ="Soft") as block: | |
| gr.Markdown(""" | |
| <div style="background-image: url('https://my.wlu.edu/Images/communications/publications/graphic-identity/300-dpi-wordmark-blue.png'); | |
| background-size: contain; | |
| background-repeat: no-repeat; | |
| background-position: center; | |
| text-align: center; | |
| height: 100px; | |
| line-height: 100px; | |
| font-size: 36px; | |
| color: white; | |
| font-family: Arial, sans-serif;"> | |
| </div> | |
| """) | |
| gr.Markdown("<h1>Fizz Chatbot</h1>") | |
| gr.Markdown("<h6><i>Disclaimer: some information may be inaccurate</i></h6>") | |
| with gr.Accordion("Parameters", open=False, visible=True, elem_classes=["parameter-accordion"]) as parameter_row: | |
| temperature = gr.Slider( | |
| minimum = 0.0, | |
| maximum = 1.0, | |
| value = 0.7, | |
| step=0.1, | |
| interactive = True, | |
| label="Temperature" | |
| ) | |
| top_p = gr.Slider( | |
| minimum = 0.0, | |
| maximum = 1.0, | |
| value = 1.0, | |
| step=0.1, | |
| interactive = True, | |
| label="Top P" | |
| ) | |
| max_new_tokens = gr.Slider( | |
| minimum = 16, | |
| maximum = 1028, | |
| value = 128, | |
| step= 32, | |
| interactive = True, | |
| label="Max tokens" | |
| ) | |
| chatbot = gr.Chatbot(label="W&L AI") | |
| message = gr.Textbox(label="", placeholder="Ask me anything about W&L here...", elem_id="input_box") | |
| submit = gr.Button("Submit Query", elem_classes="specific_button") | |
| submit.click( | |
| fn=message_and_history, | |
| inputs=[message, gr.State()], | |
| outputs=[chatbot, gr.State()] | |
| ) | |
| gr.Row([chatbot]) | |
| gr.Row([message, submit]) | |
| return block | |
| app = setup_interface() | |
| app.launch(debug=True) |