Spaces:
Sleeping
Sleeping
| import os | |
| from threading import Thread | |
| from typing import Iterator | |
| import gradio as gr | |
| import spaces | |
| import torch | |
| from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer | |
| import subprocess | |
| subprocess.run( | |
| "pip install flash-attn --no-build-isolation", | |
| env={"FLASH_ATTENTION_SKIP_CUDA_BUILD": "TRUE"}, | |
| shell=True, | |
| ) | |
| # Constants | |
| MAX_MAX_NEW_TOKENS = 2048 | |
| DEFAULT_MAX_NEW_TOKENS = 1024 | |
| MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "4096")) | |
| # Model initialization | |
| device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") | |
| model_id = "DeepMount00/Lexora-Lite-3B" | |
| tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True) | |
| model = AutoModelForCausalLM.from_pretrained( | |
| model_id, | |
| device_map="auto", | |
| torch_dtype=torch.bfloat16, | |
| attn_implementation="flash_attention_2", | |
| trust_remote_code=True, | |
| ) | |
| model.eval() | |
| # Custom CSS | |
| CUSTOM_CSS = """ | |
| /* Base styles */ | |
| * { | |
| margin: 0; | |
| padding: 0; | |
| box-sizing: border-box; | |
| } | |
| body { | |
| font-family: 'Inter', sans-serif; | |
| background-color: #f8fafc; | |
| color: #1e293b; | |
| } | |
| /* Container styles */ | |
| .container { | |
| max-width: 1000px !important; | |
| margin: auto !important; | |
| padding: 2rem !important; | |
| } | |
| /* Header styles */ | |
| .header-container { | |
| background: linear-gradient(135deg, #1e3a8a 0%, #3b82f6 100%); | |
| padding: 2.5rem; | |
| border-radius: 1rem; | |
| margin-bottom: 2rem; | |
| color: white; | |
| box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1), 0 2px 4px -1px rgba(0, 0, 0, 0.06); | |
| } | |
| .header-title { | |
| font-size: 2.5rem; | |
| font-weight: 700; | |
| margin-bottom: 1.5rem; | |
| text-align: center; | |
| letter-spacing: -0.025em; | |
| } | |
| /* Model info styles */ | |
| .model-info { | |
| background: white; | |
| padding: 1.75rem; | |
| border-radius: 0.75rem; | |
| margin-top: 1.5rem; | |
| box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); | |
| } | |
| .model-info h2 { | |
| font-size: 1.5rem; | |
| font-weight: 600; | |
| color: #1e3a8a; | |
| margin-bottom: 1rem; | |
| } | |
| .model-info p { | |
| color: #374151; | |
| line-height: 1.6; | |
| font-size: 1.1rem; | |
| } | |
| .model-info a { | |
| color: #2563eb; | |
| font-weight: 600; | |
| text-decoration: none; | |
| transition: color 0.2s; | |
| } | |
| .model-info a:hover { | |
| color: #1d4ed8; | |
| text-decoration: underline; | |
| } | |
| /* Chat container styles */ | |
| .chat-container { | |
| border: 1px solid #e5e7eb; | |
| border-radius: 1rem; | |
| background: white; | |
| box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1); | |
| margin-bottom: 2rem; | |
| } | |
| /* Message styles */ | |
| .message { | |
| padding: 1.25rem; | |
| margin: 0.75rem; | |
| border-radius: 0.5rem; | |
| font-size: 1.05rem; | |
| line-height: 1.6; | |
| } | |
| .user-message { | |
| background: #f3f4f6; | |
| border-left: 4px solid #3b82f6; | |
| } | |
| .assistant-message { | |
| background: #dbeafe; | |
| border-left: 4px solid #1d4ed8; | |
| } | |
| /* Controls container styles */ | |
| .controls-container { | |
| background: #f8fafc; | |
| padding: 1.75rem; | |
| border-radius: 0.75rem; | |
| margin-top: 1.5rem; | |
| border: 1px solid #e5e7eb; | |
| } | |
| /* Slider styles */ | |
| .slider-label { | |
| font-weight: 600; | |
| color: #374151; | |
| margin-bottom: 0.5rem; | |
| font-size: 1.05rem; | |
| } | |
| /* Button styles */ | |
| .duplicate-button { | |
| background: #2563eb !important; | |
| color: white !important; | |
| padding: 0.875rem 1.75rem !important; | |
| border-radius: 0.5rem !important; | |
| font-weight: 600 !important; | |
| font-size: 1.05rem !important; | |
| transition: all 0.2s !important; | |
| border: none !important; | |
| cursor: pointer !important; | |
| display: inline-flex !important; | |
| align-items: center !important; | |
| justify-content: center !important; | |
| text-align: center !important; | |
| box-shadow: 0 2px 4px rgba(37, 99, 235, 0.2) !important; | |
| } | |
| .duplicate-button:hover { | |
| background: #1d4ed8 !important; | |
| transform: translateY(-1px) !important; | |
| box-shadow: 0 4px 6px rgba(37, 99, 235, 0.3) !important; | |
| } | |
| /* Input field styles */ | |
| .input-textarea { | |
| border: 2px solid #e5e7eb !important; | |
| border-radius: 0.5rem !important; | |
| padding: 1rem !important; | |
| font-size: 1.05rem !important; | |
| transition: border-color 0.2s !important; | |
| } | |
| .input-textarea:focus { | |
| border-color: #3b82f6 !important; | |
| outline: none !important; | |
| box-shadow: 0 0 0 3px rgba(59, 130, 246, 0.1) !important; | |
| } | |
| /* System message styles */ | |
| .system-message { | |
| background: #f8fafc; | |
| border: 1px solid #e5e7eb; | |
| border-radius: 0.5rem; | |
| padding: 1rem; | |
| margin-bottom: 1rem; | |
| } | |
| """ | |
| # HTML Description | |
| DESCRIPTION = ''' | |
| <div class="header-container"> | |
| <h1 class="header-title">Lexora-Lite-3B</h1> | |
| <div class="model-info"> | |
| <h2>About the Model</h2> | |
| <p> | |
| Welcome to the demonstration of <a href="https://huggingface.co/DeepMount00/Lexora-Lite-3B">Lexora-Lite-3B Chat ITA</a>, | |
| currently the leading open-source large language model for the Italian language. This model represents the state-of-the-art | |
| in Italian natural language processing, combining powerful language understanding with efficient performance. | |
| </p> | |
| <p style="margin-top: 1rem;"> | |
| View its performance metrics and compare it with other models on the | |
| <a href="https://huggingface.co/spaces/FinancialSupport/open_ita_llm_leaderboard">official leaderboard</a>. | |
| </p> | |
| </div> | |
| </div> | |
| ''' | |
| def generate( | |
| message: str, | |
| chat_history: list[tuple[str, str]], | |
| system_message: str = "", | |
| max_new_tokens: int = 2048, | |
| temperature: float = 0.0001, | |
| top_p: float = 1.0, | |
| top_k: int = 50, | |
| repetition_penalty: float = 1.0, | |
| ) -> Iterator[str]: | |
| conversation = [{"role": "system", "content": system_message}] | |
| for user, assistant in chat_history: | |
| conversation.extend( | |
| [ | |
| {"role": "user", "content": user}, | |
| {"role": "assistant", "content": assistant}, | |
| ] | |
| ) | |
| conversation.append({"role": "user", "content": message}) | |
| input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt") | |
| if input_ids.shape[1] > MAX_INPUT_TOKEN_LENGTH: | |
| input_ids = input_ids[:, -MAX_INPUT_TOKEN_LENGTH:] | |
| gr.Warning(f"Trimmed input from conversation as it was longer than {MAX_INPUT_TOKEN_LENGTH} tokens.") | |
| input_ids = input_ids.to(model.device) | |
| streamer = TextIteratorStreamer(tokenizer, timeout=20.0, skip_prompt=True, skip_special_tokens=True) | |
| generate_kwargs = dict( | |
| {"input_ids": input_ids}, | |
| streamer=streamer, | |
| max_new_tokens=max_new_tokens, | |
| do_sample=True, | |
| top_p=top_p, | |
| top_k=top_k, | |
| temperature=temperature, | |
| num_beams=1, | |
| repetition_penalty=repetition_penalty, | |
| ) | |
| t = Thread(target=model.generate, kwargs=generate_kwargs) | |
| t.start() | |
| outputs = [] | |
| for text in streamer: | |
| outputs.append(text) | |
| yield "".join(outputs) | |
| def create_chat_interface(): | |
| theme = gr.themes.Soft( | |
| primary_hue="blue", | |
| secondary_hue="blue", | |
| neutral_hue="slate", | |
| font=gr.themes.GoogleFont("Inter"), | |
| radius_size=gr.themes.sizes.radius_sm, | |
| ) | |
| with gr.Blocks(css=CUSTOM_CSS, theme=theme) as demo: | |
| with gr.Column(elem_classes="container"): | |
| gr.Markdown(DESCRIPTION) | |
| with gr.Column(elem_classes="chat-container"): | |
| additional_inputs = [ | |
| gr.Textbox( | |
| value="", | |
| label="System Message", | |
| elem_classes="system-message", | |
| render=False, | |
| ), | |
| ] | |
| # Create controls without context manager | |
| controls = gr.Column(elem_classes="controls-container") | |
| with controls: | |
| additional_inputs.extend([ | |
| gr.Slider( | |
| label="Maximum New Tokens", | |
| minimum=1, | |
| maximum=MAX_MAX_NEW_TOKENS, | |
| step=1, | |
| value=DEFAULT_MAX_NEW_TOKENS, | |
| elem_classes="slider-label", | |
| ), | |
| gr.Slider( | |
| label="Temperature", | |
| minimum=0, | |
| maximum=4.0, | |
| step=0.1, | |
| value=0.001, | |
| elem_classes="slider-label", | |
| ), | |
| gr.Slider( | |
| label="Top-p (Nucleus Sampling)", | |
| minimum=0.05, | |
| maximum=1.0, | |
| step=0.05, | |
| value=1.0, | |
| elem_classes="slider-label", | |
| ), | |
| gr.Slider( | |
| label="Top-k", | |
| minimum=1, | |
| maximum=1000, | |
| step=1, | |
| value=50, | |
| elem_classes="slider-label", | |
| ), | |
| gr.Slider( | |
| label="Repetition Penalty", | |
| minimum=1.0, | |
| maximum=2.0, | |
| step=0.05, | |
| value=1.0, | |
| elem_classes="slider-label", | |
| ), | |
| ]) | |
| chat_interface = gr.ChatInterface( | |
| fn=generate, | |
| additional_inputs=additional_inputs, | |
| examples=[ | |
| ["Ciao! Come stai?"], | |
| ["Raccontami una breve storia."], | |
| ["Qual è il tuo piatto italiano preferito?"], | |
| ], | |
| cache_examples=False, | |
| ) | |
| gr.DuplicateButton( | |
| value="Duplicate Space for Private Use", | |
| elem_classes="duplicate-button", | |
| elem_id="duplicate-button", | |
| ) | |
| return demo | |
| if __name__ == "__main__": | |
| demo = create_chat_interface() | |
| demo.queue(max_size=20).launch() |