Update app.py

app.py

@@ -1,184 +1,551 @@
+import argparse
+from collections import defaultdict
+import datetime
+import json
 import os
-import gradio as gr
-import numpy as np
-import random
-import torch
-import subprocess
 import time
+import uuid
+
+import gradio as gr
 import requests
-import json
 
-import base64
-from io import BytesIO
-from PIL import Image
-from huggingface_hub import login
-from huggingface_hub.utils import (
-    HfFolder
+from fastchat.conversation import (
+    Conversation,
+    compute_skip_echo_len,
+    SeparatorStyle,
 )
+from fastchat.constants import LOGDIR
+from fastchat.utils import (
+    build_logger,
+    server_error_msg,
+    violates_moderation,
+    moderation_msg,
+)
+from fastchat.serve.gradio_patch import Chatbot as grChatbot
+from fastchat.serve.gradio_css import code_highlight_css
 
-myip = os.environ["myip"]
-myport = os.environ["myport"]
-
-url = f"http://{myip}:{myport}"
 
-queue_size = 0
+logger = build_logger("gradio_web_server", "gradio_web_server.log")
 
-def displayTextBox():
-    global queue_size
-    if queue_size > 4:
-        return [gr.update(visible=False), gr.update(visible=True)]
-    elif queue_size <= 4:
-        return [gr.update(visible=True), gr.update(visible=False)]       
+headers = {"User-Agent": "NeuralChat Client"}
 
+no_change_btn = gr.Button.update()
+enable_btn = gr.Button.update(interactive=True)
+disable_btn = gr.Button.update(interactive=False)
 
-def set_msg():
-    global queue_size
-    que_high_msg = "The current traffic is high with " + str(queue_size) + " in the queue. Please wait a moment."
-    que_normal_msg = "The current traffic is not high. You can submit your job now."
-    
-    if queue_size > int(os.environ["max_queue_size"]):
-        return que_high_msg
-    else:
-        return que_normal_msg
+controller_url = None
+enable_moderation = False
 
+conv_template_bf16 = Conversation(
+    system="A chat between a curious human and an artificial intelligence assistant. "
+           "The assistant gives helpful, detailed, and polite answers to the human's questions.",
+    roles=("Human", "Assistant"),
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.SINGLE,
+    sep="\n",
+    sep2="</s>",
+)
 
-def img2img_generate(source_img, prompt, steps=25, strength=0.75, seed=42, guidance_scale=7.5):
 
-    print('image-to-image')
-    print("prompt: ", prompt)
-    print("steps: ", steps)
-    buffered = BytesIO()
-    source_img.save(buffered, format="JPEG")
-    img_b64 = base64.b64encode(buffered.getvalue())
-    timestamp = int(time.time()*1000)
+def set_global_vars(controller_url_, enable_moderation_):
+    global controller_url, enable_moderation
+    controller_url = controller_url_
+    enable_moderation = enable_moderation_
 
-    data = {"source_img": img_b64.decode(), "prompt": prompt, "steps": steps,
-            "guidance_scale": guidance_scale, "seed": seed, "strength": strength,
-            "task_type": "1",
-            "timestamp": timestamp, "user": os.environ.get("token", "")}
 
-    start_time = time.time()
-    global queue_size
-    queue_size = queue_size + 1
-    resp = requests.post(url, data=json.dumps(data))
-    queue_size = queue_size - 1
+def get_conv_log_filename():
+    t = datetime.datetime.now()
+    name = os.path.join(LOGDIR, f"{t.year}-{t.month:02d}-{t.day:02d}-conv.json")
+    return name
 
-    try:
-        img_str = json.loads(resp.text)["img_str"]
-        print("Compute node: ", json.loads(resp.text)["ip"])
-    except:
-        print('No inference result. Please check server connection')
-        return None
-
-    img_byte = base64.b64decode(img_str)
-    img_io = BytesIO(img_byte)  # convert image to file-like object
-    img = Image.open(img_io)   # img is now PIL Image object
-    print("elapsed time: ", time.time() - start_time)
-    return img
-
-
-def txt2img_generate(prompt, steps=25, seed=42, guidance_scale=7.5):
-
-    print('text-to-image')
-    print("prompt: ", prompt)
-    print("steps: ", steps)
-    timestamp = int(time.time()*1000)
-    data = {"prompt": prompt,
-        "steps": steps, "guidance_scale": guidance_scale, "seed": seed,
-        "task_type": "0",
-        "timestamp": timestamp, "user": os.environ.get("token", "")}
-    start_time = time.time()
-    global queue_size
-    queue_size = queue_size + 1
-    resp = requests.post(url, data=json.dumps(data))
-    queue_size = queue_size - 1
-    try:
-        img_str = json.loads(resp.text)["img_str"]
-        print("Compute node: ", json.loads(resp.text)["ip"])
-    except:
-        print('No inference result. Please check server connection')
-        return None
-
-    img_byte = base64.b64decode(img_str)
-    img_io = BytesIO(img_byte)  # convert image to file-like object
-    img = Image.open(img_io)   # img is now PIL Image object
-    print("elapsed time: ", time.time() - start_time)
 
-    return img
+def get_model_list(controller_url):
+    ret = requests.post(controller_url + "/refresh_all_workers")
+    assert ret.status_code == 200
+    ret = requests.post(controller_url + "/list_models")
+    models = ret.json()["models"]
+    logger.info(f"Models: {models}")
+    return models
 
 
-md = """
-This demo shows the accelerated inference performance of a Stable Diffusion model on **Intel Xeon Gold 64xx (4th Gen Intel Xeon Scalable Processors codenamed Sapphire Rapids)**. Try it and generate photorealistic images from text! Please note that the demo is in **preview** under limited HW resources. We are committed to continue improving the demo and happy to hear your feedbacks. Thanks for your trying！
-You may also want to try creating your own Stable Diffusion with few-shot fine-tuning. Please refer to our <a href=\"https://medium.com/intel-analytics-software/personalized-stable-diffusion-with-few-shot-fine-tuning-on-a-single-cpu-f01a3316b13\">blog</a> and <a href=\"https://github.com/intel/neural-compressor/tree/master/examples/pytorch/diffusion_model/diffusers/textual_inversion\">code</a> available in <a href=\"https://github.com/intel/neural-compressor\">**Intel Neural Compressor**</a> and <a href=\"https://github.com/huggingface/diffusers\">**Hugging Face Diffusers**</a>.
+get_window_url_params = """
+function() {
+    const params = new URLSearchParams(window.location.search);
+    url_params = Object.fromEntries(params);
+    console.log("url_params", url_params);
+    return url_params;
+    }
 """
 
-legal = """
-Performance varies by use, configuration and other factors. Learn more at www.Intel.com/PerformanceIndex. Performance results are based on testing as of dates shown in configurations and may not reflect all publicly available updates.  See backup for configuration details.  No product or component can be absolutely secure.
-© Intel Corporation. Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries. Other names and brands may be claimed as the property of others.
-"""
 
-details = """
-4th Gen Intel Xeon Scalable Processor Inference. Test by Intel on 01/06/2023. 1 node, 1S, Intel(R) Xeon(R) Gold 64xx CPU @ 3.0GHz 32 cores and software with 512GB (8x64GB DDR5 4800 MT/s [4800 MT/s]), microcode 0x2a000080, HT on, Turbo on, Ubuntu 22.04.1 LTS, 5.15.0-1026-aws, 200G Amazon Elastic Block Store. Multiple nodes connected with Elastic Network Adapter (ENA). PyTorch Nightly build (2.0.0.dev20230105+cpu), Transformers 4.25.1, Diffusers 0.11.1, oneDNN v2.7.2.
-"""
-
-css = '''
-    .instruction{position: absolute; top: 0;right: 0;margin-top: 0px !important}
-    .arrow{position: absolute;top: 0;right: -110px;margin-top: -8px !important}
-    #component-4, #component-3, #component-10{min-height: 0}
-    .duplicate-button img{margin: 0}
-    #mdStyle{font-size: 0.6rem}
-    .generating.svelte-1w9161c { border: none }
-    #txtGreenStyle {border: 2px solid #32ec48;}
-    #txtOrangeStyle {border: 2px solid #e77718;}
-'''
-
-random_seed = random.randint(0, 2147483647)
-
-with gr.Blocks(css=css) as demo:
-    gr.Markdown("# Stable Diffusion Inference Demo on 4th Gen Intel Xeon Scalable Processors")
-    gr.Markdown(md)
-
-    textBoxGreen = gr.Textbox(set_msg, every=1, label='Real-time Jobs in Queue', elem_id='txtGreenStyle', visible=True)
-    textBoxOrange = gr.Textbox(set_msg, every=1, label='Real-time Jobs in Queue', elem_id='txtOrangeStyle', visible=False)
-    textBoxGreen.change(displayTextBox, outputs = [textBoxGreen, textBoxOrange])
-
-    with gr.Tab("Text-to-Image"):
-        with gr.Row(visible=True) as text_to_image:
-            with gr.Column():
-                prompt = gr.inputs.Textbox(label='Prompt', default='a photo of an astronaut riding a horse on mars')
-                inference_steps = gr.inputs.Slider(1, 100, label='Inference Steps - increase the steps for better quality (e.g., avoiding black image) ', default=20, step=1)
-                seed = gr.inputs.Slider(0, 2147483647, label='Seed', default=random_seed, step=1)
-                guidance_scale = gr.inputs.Slider(1.0, 20.0, label='Guidance Scale - how much the prompt will influence the results', default=7.5, step=0.1)
-                txt2img_button = gr.Button("Generate Image")
-
-            with gr.Column():
-                result_image = gr.Image()
+def load_demo_single(models, url_params):
+    dropdown_update = gr.Dropdown.update(visible=True)
+    if "model" in url_params:
+        model = url_params["model"]
+        if model in models:
+            dropdown_update = gr.Dropdown.update(value=model, visible=True)
+
+    state = None
+    return (
+        state,
+        dropdown_update,
+        gr.Chatbot.update(visible=True),
+        gr.Textbox.update(visible=True),
+        gr.Button.update(visible=True),
+        gr.Row.update(visible=True),
+        gr.Accordion.update(visible=False),
+    )
+
+
+def load_demo(url_params, request: gr.Request):
+    logger.info(f"load_demo. ip: {request.client.host}. params: {url_params}")
+    return load_demo_single(models, url_params)
+
+
+def vote_last_response(state, vote_type, model_selector, request: gr.Request):
+    with open(get_conv_log_filename(), "a") as fout:
+        data = {
+            "tstamp": round(time.time(), 4),
+            "type": vote_type,
+            "model": model_selector,
+            "state": state.dict(),
+            "ip": request.client.host,
+        }
+        fout.write(json.dumps(data) + "\n")
+
+
+def upvote_last_response(state, model_selector, request: gr.Request):
+    logger.info(f"upvote. ip: {request.client.host}")
+    vote_last_response(state, "upvote", model_selector, request)
+    return ("",) + (disable_btn,) * 3
+
+
+def downvote_last_response(state, model_selector, request: gr.Request):
+    logger.info(f"downvote. ip: {request.client.host}")
+    vote_last_response(state, "downvote", model_selector, request)
+    return ("",) + (disable_btn,) * 3
+
+
+def flag_last_response(state, model_selector, request: gr.Request):
+    logger.info(f"flag. ip: {request.client.host}")
+    vote_last_response(state, "flag", model_selector, request)
+    return ("",) + (disable_btn,) * 3
+
+
+def regenerate(state, request: gr.Request):
+    logger.info(f"regenerate. ip: {request.client.host}")
+    state.messages[-1][-1] = None
+    state.skip_next = False
+    return (state, state.to_gradio_chatbot(), "") + (disable_btn,) * 5
+
+
+def clear_history(request: gr.Request):
+    logger.info(f"clear_history. ip: {request.client.host}")
+    state = None
+    return (state, [], "") + (disable_btn,) * 5
+
+
+def add_text(state, text, request: gr.Request):
+    logger.info(f"add_text. ip: {request.client.host}. len: {len(text)}")
+
+    if state is None:
+        state = conv_template_bf16.copy()
+
+    if len(text) <= 0:
+        state.skip_next = True
+        return (state, state.to_gradio_chatbot(), "") + (no_change_btn,) * 5
+    if enable_moderation:
+        flagged = violates_moderation(text)
+        if flagged:
+            logger.info(f"violate moderation. ip: {request.client.host}. text: {text}")
+            state.skip_next = True
+            return (state, state.to_gradio_chatbot(), moderation_msg) + (
+                no_change_btn,
+            ) * 5
+
+    text = text[:1536]  # Hard cut-off
+    state.append_message(state.roles[0], text)
+    state.append_message(state.roles[1], None)
+    state.skip_next = False
+    return (state, state.to_gradio_chatbot(), "") + (disable_btn,) * 5
+
+
+def post_process_code(code):
+    sep = "\n```"
+    if sep in code:
+        blocks = code.split(sep)
+        if len(blocks) % 2 == 1:
+            for i in range(1, len(blocks), 2):
+                blocks[i] = blocks[i].replace("\\_", "_")
+        code = sep.join(blocks)
+    return code
+
+
+def http_bot(state, model_selector, temperature, max_new_tokens, request: gr.Request):
+    logger.info(f"http_bot. ip: {request.client.host}")
+    start_tstamp = time.time()
+    model_name = model_selector
+    temperature = float(temperature)
+    max_new_tokens = int(max_new_tokens)
+
+    if state.skip_next:
+        # This generate call is skipped due to invalid inputs
+        yield (state, state.to_gradio_chatbot()) + (no_change_btn,) * 5
+        return
+
+    if len(state.messages) == state.offset + 2:
+        # First round of conversation
+        new_state = conv_template_bf16.copy()
+        new_state.conv_id = uuid.uuid4().hex
+        new_state.model_name = state.model_name or model_selector
+        new_state.append_message(new_state.roles[0], state.messages[-2][1])
+        new_state.append_message(new_state.roles[1], None)
+        state = new_state
+
+    # Query worker address
+    ret = requests.post(
+        controller_url + "/get_worker_address", json={"model": model_name}
+    )
+    worker_addr = ret.json()["address"]
+    logger.info(f"model_name: {model_name}, worker_addr: {worker_addr}")
+
+    # No available worker
+    if worker_addr == "":
+        state.messages[-1][-1] = server_error_msg
+        yield (
+            state,
+            state.to_gradio_chatbot(),
+            disable_btn,
+            disable_btn,
+            disable_btn,
+            enable_btn,
+            enable_btn,
+        )
+        return
+
+    # Construct prompt
+    prompt = state.get_prompt()
+    skip_echo_len = compute_skip_echo_len(model_name, state, prompt)
+
+    # Make requests
+    pload = {
+        "model": model_name,
+        "prompt": prompt,
+        "temperature": temperature,
+        "max_new_tokens": max_new_tokens,
+        "stop": "</s>"
+    }
+    logger.info(f"==== request ====\n{pload}")
 
+    start_time = time.time()
 
-    with gr.Tab("Image-to-Image text-guided generation"):
-        with gr.Row(visible=True) as image_to_image:
-            with gr.Column():
-                source_img = gr.Image(source="upload", type="pil", value="https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg")
-                # source_img = gr.Image(source="upload", type="pil")
-                prompt_2 = gr.inputs.Textbox(label='Prompt', default='A fantasy landscape, trending on artstation')
-                inference_steps_2 = gr.inputs.Slider(1, 100, label='Inference Steps - increase the steps for better quality (e.g., avoiding black image) ', default=20, step=1)
-                seed_2 = gr.inputs.Slider(0, 2147483647, label='Seed', default=random_seed, step=1)
-                guidance_scale_2 = gr.inputs.Slider(1.0, 20.0, label='Guidance Scale - how much the prompt will influence the results', default=7.5, step=0.1)
-                strength = gr.inputs.Slider(0.0, 1.0, label='Strength - adding more noise to it the larger the strength', default=0.75, step=0.01)
-                img2img_button = gr.Button("Generate Image")
+    state.messages[-1][-1] = "▌"
+    yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
 
-            with gr.Column():
-                result_image_2 = gr.Image()
+    try:
+        # Stream output
+        response = requests.post(
+            worker_addr + "/worker_generate_stream",
+            headers=headers,
+            json=pload,
+            stream=True,
+            timeout=20,
+        )
+        for chunk in response.iter_lines(decode_unicode=False, delimiter=b"\0"):
+            if chunk:
+                data = json.loads(chunk.decode())
+                if data["error_code"] == 0:
+                    output = data["text"][skip_echo_len:].strip()
+                    output = post_process_code(output)
+                    state.messages[-1][-1] = output + "▌"
+                    yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
+                else:
+                    output = data["text"] + f" (error_code: {data['error_code']})"
+                    state.messages[-1][-1] = output
+                    yield (state, state.to_gradio_chatbot()) + (
+                        disable_btn,
+                        disable_btn,
+                        disable_btn,
+                        enable_btn,
+                        enable_btn,
+                    )
+                    return
+                time.sleep(0.02)
+    except requests.exceptions.RequestException as e:
+        state.messages[-1][-1] = server_error_msg + f" (error_code: 4)"
+        yield (state, state.to_gradio_chatbot()) + (
+            disable_btn,
+            disable_btn,
+            disable_btn,
+            enable_btn,
+            enable_btn,
+        )
+        return
+
+    finish_tstamp = time.time() - start_time
+    elapsed_time = "\n✅generation elapsed time: {}s".format(round(finish_tstamp, 4))
+
+    state.messages[-1][-1] = state.messages[-1][-1][:-1] + elapsed_time
+    yield (state, state.to_gradio_chatbot()) + (enable_btn,) * 5
+
+    logger.info(f"{output}")
+
+    with open(get_conv_log_filename(), "a") as fout:
+        data = {
+            "tstamp": round(finish_tstamp, 4),
+            "type": "chat",
+            "model": model_name,
+            "gen_params": {
+                "temperature": temperature,
+                "max_new_tokens": max_new_tokens,
+            },
+            "start": round(start_tstamp, 4),
+            "finish": round(start_tstamp, 4),
+            "state": state.dict(),
+            "ip": request.client.host,
+        }
+        fout.write(json.dumps(data) + "\n")
+
+
+block_css = (
+    code_highlight_css
+    + """
+pre {
+    white-space: pre-wrap;       /* Since CSS 2.1 */
+    white-space: -moz-pre-wrap;  /* Mozilla, since 1999 */
+    white-space: -pre-wrap;      /* Opera 4-6 */
+    white-space: -o-pre-wrap;    /* Opera 7 */
+    word-wrap: break-word;       /* Internet Explorer 5.5+ */
+}
+#notice_markdown th {
+    display: none;
+}
+
+#notice_markdown {
+text-align: center;
+    background: #874bec;
+    padding: 1%;
+}
+
+#notice_markdown h1, #notice_markdown h4 {
+    color: #fff;
+    margin-top: 0;
+}
+
+gradio-app {
+    background: linear-gradient(to bottom, #ba97d8, #5400ff) !important;
+    padding: 3%;
+}
+
+.gradio-container {
+    margin: 0 auto !important;
+    width: 70% !important;
+    padding: 0 !important;
+}
+
+#chatbot {
+    border-style: solid;
+    overflow: visible;
+    margin: 1% 4%;
+    width: 90%;
+    box-shadow: 0 15px 15px -5px rgba(0, 0, 0, 0.2);
+    border: 1px solid #ddd;
+}
+
+#text-box-style, #btn-style {
+    width: 90%;
+    margin: 1% 4%;
+}
+
+
+.user, .bot {
+    width: 80% !important;
+    
+}
+
+.bot {
+    white-space: pre-wrap !important;  
+    line-height: 1.3 !important;
+    display: flex;
+    flex-direction: column;
+    justify-content: flex-start;
+
+}
+
+#btn-send-style {
+    background: rgb(0, 180, 50);
+    color: #fff;
+    }
+
+#btn-list-style {
+    background: #eee0;
+    border: 1px solid #691ef7;
+}        
+"""
+)
 
-    txt2img_button.click(fn=txt2img_generate, inputs=[prompt, inference_steps, seed, guidance_scale], outputs=[result_image])
 
-    img2img_button.click(fn=img2img_generate, inputs=[source_img, prompt_2, inference_steps_2, strength, seed_2, guidance_scale_2], outputs=result_image_2)
+def build_single_model_ui(models):
+    notice_markdown = """
+# 🤖 NeuralChat
 
-    gr.Markdown("**Additional Test Configuration Details:**", elem_id='mdStyle')
-    gr.Markdown(details, elem_id='mdStyle')
+#### deployed on 4th Gen Intel Xeon Scalable Processors codenamed Sapphire Rapids
 
-    gr.Markdown("**Notices and Disclaimers:**", elem_id='mdStyle')
-    gr.Markdown(legal, elem_id='mdStyle')
+"""
 
-demo.queue(max_size=int(os.environ["max_job_size"]), concurrency_count=int(os.environ["max_job_size"])).launch(debug=True, show_api=False)
+    learn_more_markdown = """
+"""
 
+    state = gr.State()
+    notice = gr.Markdown(notice_markdown, elem_id="notice_markdown")
+
+    with gr.Row(elem_id="model_selector_row", visible=False):
+        model_selector = gr.Dropdown(
+            choices=models,
+            value=models[0] if len(models) > 0 else "",
+            interactive=True,
+            show_label=False,
+        ).style(container=False)
+
+    chatbot = grChatbot(elem_id="chatbot", visible=False).style(height=550)
+    with gr.Row(elem_id="text-box-style"):
+        with gr.Column(scale=20):
+            textbox = gr.Textbox(
+                show_label=False,
+                placeholder="Enter text and press ENTER",
+                visible=False,
+            ).style(container=False)
+        with gr.Column(scale=1, min_width=50):
+            send_btn = gr.Button(value="Send", visible=False, elem_id="btn-send-style")
+
+    with gr.Row(visible=False, elem_id="btn-style") as button_row:
+        upvote_btn = gr.Button(value="👍  Upvote", interactive=False, elem_id="btn-list-style")
+        downvote_btn = gr.Button(value="👎  Downvote", interactive=False, elem_id="btn-list-style")
+        flag_btn = gr.Button(value="⚠️  Flag", interactive=False, visible=False, elem_id="btn-list-style")
+        # stop_btn = gr.Button(value="⏹️  Stop Generation", interactive=False)
+        regenerate_btn = gr.Button(value="🔄  Regenerate", interactive=False, elem_id="btn-list-style")
+        clear_btn = gr.Button(value="🗑️  Clear history", interactive=False, elem_id="btn-list-style")
+
+    with gr.Accordion("Parameters", open=False, visible=False) as parameter_row:
+        temperature = gr.Slider(
+            minimum=0.0,
+            maximum=1.0,
+            value=0.95,
+            step=0.1,
+            interactive=True,
+            label="Temperature",
+        )
+        max_output_tokens = gr.Slider(
+            minimum=0,
+            maximum=1024,
+            value=512,
+            step=64,
+            interactive=True,
+            label="Max output tokens",
+        )
+
+    gr.Markdown(learn_more_markdown)
+
+    # Register listeners
+    btn_list = [upvote_btn, downvote_btn, flag_btn, regenerate_btn, clear_btn]
+    upvote_btn.click(
+        upvote_last_response,
+        [state, model_selector],
+        [textbox, upvote_btn, downvote_btn, flag_btn],
+    )
+    downvote_btn.click(
+        downvote_last_response,
+        [state, model_selector],
+        [textbox, upvote_btn, downvote_btn, flag_btn],
+    )
+    flag_btn.click(
+        flag_last_response,
+        [state, model_selector],
+        [textbox, upvote_btn, downvote_btn, flag_btn],
+    )
+    regenerate_btn.click(regenerate, state, [state, chatbot, textbox] + btn_list).then(
+        http_bot,
+        [state, model_selector, temperature, max_output_tokens],
+        [state, chatbot] + btn_list,
+    )
+    clear_btn.click(clear_history, None, [state, chatbot, textbox] + btn_list)
+
+    model_selector.change(clear_history, None, [state, chatbot, textbox] + btn_list)
+
+    textbox.submit(
+        add_text, [state, textbox], [state, chatbot, textbox] + btn_list
+    ).then(
+        http_bot,
+        [state, model_selector, temperature, max_output_tokens],
+        [state, chatbot] + btn_list,
+    )
+    send_btn.click(
+        add_text, [state, textbox], [state, chatbot, textbox] + btn_list
+    ).then(
+        http_bot,
+        [state, model_selector, temperature, max_output_tokens],
+        [state, chatbot] + btn_list,
+    )
+
+    return state, model_selector, chatbot, textbox, send_btn, button_row, parameter_row
+
+
+def build_demo(models):
+    with gr.Blocks(
+        title="NeuralChat · Intel",
+        theme=gr.themes.Base(),
+        css=block_css,
+    ) as demo:
+        url_params = gr.JSON(visible=False)
+
+        (
+            state,
+            model_selector,
+            chatbot,
+            textbox,
+            send_btn,
+            button_row,
+            parameter_row,
+        ) = build_single_model_ui(models)
+
+        if model_list_mode == "once":
+            demo.load(
+                load_demo,
+                [url_params],
+                [
+                    state,
+                    model_selector,
+                    chatbot,
+                    textbox,
+                    send_btn,
+                    button_row,
+                    parameter_row,
+                ],
+                _js=get_window_url_params,
+            )
+        else:
+            raise ValueError(f"Unknown model list mode: {model_list_mode}")
+
+    return demo
+
+
+if __name__ == "__main__":
+
+    controller_url = "http://mlp-dgx-01.sh.intel.com:21001"
+    host = "mlp-dgx-01.sh.intel.com"
+    # port = "mlp-dgx-01.sh.intel.com"
+    concurrency_count = 10
+    model_list_mode = "once"
+    share = True
+    moderate = False
+
+    set_global_vars(controller_url, moderate)
+    models = get_model_list(controller_url)
+
+    demo = build_demo(models)
+    demo.queue(
+        concurrency_count=concurrency_count, status_update_rate=10, api_open=False
+    ).launch(
+        server_name=host, share=share, max_threads=200
+    )