update

README.md

@@ -0,0 +1,10 @@
+---
+title: LLM As Serve
+emoji: 🦙🚀
+sdk: gradio
+app_file: app.py
+pinned: true
+license: apache-2.0
+colorFrom: yellow
+colorTo: green
+---

app.py

@@ -0,0 +1,524 @@
+import time
+import json
+import re
+import os
+from os import listdir
+from os.path import isfile, join
+import gradio as gr
+import args
+import global_vars
+from chats import central
+from transformers import AutoModelForCausalLM
+from miscs.styles import MODEL_SELECTION_CSS
+from miscs.js import GET_LOCAL_STORAGE, UPDATE_LEFT_BTNS_STATE
+from utils import get_chat_interface, get_chat_manager, get_global_context
+
+ex_file = open("examples.txt", "r")
+examples = ex_file.read().split("\n")
+ex_btns = []
+
+chl_file = open("channels.txt", "r")
+channels = chl_file.read().split("\n")
+channel_btns = []
+
+global_vars.initialize_globals()
+
+response_configs = [
+    f"configs/response_configs/{f}"
+    for f in listdir("configs/response_configs")
+    if isfile(join("configs/response_configs", f))
+]
+
+summarization_configs = [
+    f"configs/summarization_configs/{f}"
+    for f in listdir("configs/summarization_configs")
+    if isfile(join("configs/summarization_configs", f))
+]
+
+model_info = json.load(open("model_cards.json"))
+
+def channel_num(btn_title):
+    choice = 0
+
+    for idx, channel in enumerate(channels):
+        if channel == btn_title:
+            choice = idx
+
+    return choice
+
+
+def set_chatbot(btn, ld, state):
+    choice = channel_num(btn)
+
+    res = [state["ppmanager_type"].from_json(json.dumps(ppm_str)) for ppm_str in ld]
+    empty = len(res[choice].pingpongs) == 0
+    return (
+        res[choice].build_uis(), 
+        choice, 
+        gr.update(visible=empty), 
+        gr.update(interactive=not empty)
+    )
+
+
+def set_example(btn):
+    return btn, gr.update(visible=False)
+
+
+def set_popup_visibility(ld, example_block):
+    return example_block
+
+
+def move_to_second_view(btn):
+    info = model_info[btn]
+
+    guard_vram = 5 * 1024.
+    vram_req_full = int(info["vram(full)"]) + guard_vram
+    vram_req_8bit = int(info["vram(8bit)"]) + guard_vram
+    vram_req_4bit = int(info["vram(4bit)"]) + guard_vram
+    
+    load_mode_list = []
+    
+    return (
+        gr.update(visible=False),
+        gr.update(visible=True),
+        info["thumb"],
+        f"## {btn}",
+        f"**Parameters**\n: Approx. {info['parameters']}",
+        f"**🤗 Hub(base)**\n: {info['hub(base)']}",
+        f"**🤗 Hub(LoRA)**\n: {info['hub(ckpt)']}",
+        info['desc'],
+        f"""**Min VRAM requirements** :
+|             half precision            |             load_in_8bit           |              load_in_4bit          | 
+| ------------------------------------- | ---------------------------------- | ---------------------------------- | 
+|   {round(vram_req_full/1024., 1)}GiB  | {round(vram_req_8bit/1024., 1)}GiB | {round(vram_req_4bit/1024., 1)}GiB |
+""",
+        info['default_gen_config'],
+        info['example1'],
+        info['example2'],
+        info['example3'],
+        info['example4'],
+        "",
+    )
+
+
+def move_to_first_view():
+    return (
+        gr.update(visible=True), 
+        gr.update(visible=False),
+        ""
+    )
+
+
+def get_model_num(
+    model_name
+):
+    model_num = 0
+    re_tag = re.compile(r'<[^>]+>')
+    model_name = re_tag.sub('', model_name).strip()
+    print(model_name)    
+    
+    for idx, item in enumerate(global_vars.models):
+        if item["model_name"] == model_name:
+            model_num = idx
+            print(idx)
+            break
+            
+    return "Download completed!", model_num
+
+def move_to_third_view(model_num):
+    gen_config = global_vars.models[model_num]["gen_config"]
+
+    return (
+        "Preparation done!",
+        gr.update(visible=False),
+        gr.update(visible=True),
+        gr.update(label=global_vars.models[model_num]["model_type"]),
+        {
+            "ppmanager_type": global_vars.models[model_num]["chat_manager"],
+            "model_type": global_vars.models[model_num]["model_type"],
+        },
+        get_global_context(global_vars.models[model_num]["model_type"]),
+        gen_config.temperature,
+        gen_config.top_p,
+        gen_config.top_k,
+        gen_config.repetition_penalty,
+        gen_config.max_new_tokens,
+        gen_config.num_beams,
+        gen_config.use_cache,
+        gen_config.do_sample,
+        gen_config.eos_token_id,
+        gen_config.pad_token_id,
+    )
+
+
+def toggle_inspector(view_selector):
+    if view_selector == "with context inspector":
+        return gr.update(visible=True)
+    else:
+        return gr.update(visible=False)
+
+
+def reset_chat(idx, ld, state):
+    res = [state["ppmanager_type"].from_json(json.dumps(ppm_str)) for ppm_str in ld]
+    res[idx].pingpongs = []
+        
+    return (
+        "",
+        [],
+        str(res),
+        gr.update(visible=True),
+        gr.update(interactive=False),
+    )
+
+def rollback_last(idx, ld, state):
+    res = [state["ppmanager_type"].from_json(json.dumps(ppm_str)) for ppm_str in ld]
+    last_user_message = res[idx].pingpongs[-1].ping
+    res[idx].pingpongs = res[idx].pingpongs[:-1]
+    
+    return (
+        last_user_message,
+        res[idx].build_uis(),
+        str(res),
+        gr.update(interactive=False)
+    )
+
+with gr.Blocks(css=MODEL_SELECTION_CSS, theme='gradio/soft') as demo:
+    with gr.Column() as model_choice_view:
+        gr.Markdown("# Choose a Model", elem_classes=["center"])
+      
+        with gr.Row(elem_id="container"):
+            with gr.Column():
+                gr.Markdown("""This application is built and provided for anyone who wants to try out open source Large Language Models for free. All the provided models are pre-downloaded and pre-loaded to maximize your experience. This application is hosted on [jarvislabs.ai](https://jarvislabs.ai/) with 3 x A6000 VM instance. This demo will be hosted until 13/07/2023, but you can run the same application on [jarvislabs.ai](https://jarvislabs.ai/) with arbitrary GPU options of your choice. Also, if you can run the same application on your own environment, be sure to check out the [project repository](https://github.com/deep-diver/LLM-As-Chatbot) for any further information.
+
+From this page, choose a model that you would like to try out. By selecting a model, you will see more detailed description of the model in a separate page. Also note that this page will appear whenever you refresh your browser tab. """)
+                with gr.Row(elem_classes=["sub-container"]):
+                    # with gr.Column(min_width=20):
+                    #     llama_deus_7b = gr.Button("llama-deus-7b", elem_id="llama-deus-7b", elem_classes=["square"])
+                    #     gr.Markdown("LLaMA Deus", elem_classes=["center"])                    
+
+                    with gr.Column(min_width=20):                        
+                        baize_7b = gr.Button("baize-7b", elem_id="baize-7b", elem_classes=["square"])
+                        gr.Markdown("Baize", elem_classes=["center"])                            
+                        
+#                     with gr.Column(min_width=20):
+#                         koalpaca = gr.Button("koalpaca", elem_id="koalpaca", elem_classes=["square"])
+#                         gr.Markdown("koalpaca", elem_classes=["center"])                        
+                        
+                    # with gr.Column(min_width=20):
+                    #     evolinstruct_vicuna_13b = gr.Button("evolinstruct-vicuna-13b", elem_id="evolinstruct-vicuna-13b", elem_classes=["square"])
+                    #     gr.Markdown("EvolInstruct Vicuna", elem_classes=["center"])                      
+                        
+                    with gr.Column(min_width=20):
+                        guanaco_7b = gr.Button("guanaco-7b", elem_id="guanaco-7b", elem_classes=["square"])
+                        gr.Markdown("Guanaco", elem_classes=["center"])
+                        
+                    # with gr.Column(min_width=20):
+                    #     nous_hermes_13b = gr.Button("nous-hermes-13b", elem_id="nous-hermes-13b", elem_classes=["square"])
+                    #     gr.Markdown("Nous Hermes", elem_classes=["center"])                        
+                        
+                progress_view = gr.Textbox(label="Progress")
+
+    with gr.Column(visible=False) as model_review_view:
+        gr.Markdown("# Confirm the chosen model", elem_classes=["center"])
+
+        with gr.Column(elem_id="container2"):
+            gr.Markdown("""The model is pre-downloaded and pre-loaded for your convenience in this demo application, so you don't need to worry about the `VRAM requirements`. It is there just as a reference. Also, proper `GenerationConfig` is selected and fixed, but you can adjust some of the hyper-parameters once you enter the chatting mode.
+
+Before deciding which model to use, you can expand `Example showcases` to see some of the recorded example pairs of question and answer. It will help you understanding better which model suits you well. Then, click `Confirm` button to enter the chatting mode. If you click `Back` button or refresh the browser tab, the model selection page will appear. 
+""")
+
+            with gr.Row():
+                model_image = gr.Image(None, interactive=False, show_label=False)
+                with gr.Column():
+                    model_name = gr.Markdown("**Model name**")
+                    model_desc = gr.Markdown("...")                        
+                    model_params = gr.Markdown("Parameters\n: ...")             
+                    model_base = gr.Markdown("🤗 Hub(base)\n: ...")
+                    model_ckpt = gr.Markdown("🤗 Hub(LoRA)\n: ...")
+                    model_vram = gr.Markdown(f"""**Minimal VRAM requirement** :
+|          half precision        |        load_in_8bit       |         load_in_4bit      | 
+| ------------------------------ | ------------------------- | ------------------------- | 
+|   {round(7830/1024., 1)}GiB    | {round(5224/1024., 1)}GiB | {round(4324/1024., 1)}GiB |
+""")
+                    model_thumbnail_tiny = gr.Textbox("", visible=False)
+
+            with gr.Column():
+                gen_config_path = gr.Dropdown(
+                    response_configs,
+                    value=response_configs[0],
+                    interactive=False,
+                    label="Gen Config(response)",
+                )
+
+                with gr.Accordion("Example showcases", open=False):
+                    with gr.Tab("Ex1"):
+                        example_showcase1 = gr.Chatbot(
+                            [("hello", "world"), ("damn", "good")]
+                        )
+                    with gr.Tab("Ex2"):
+                        example_showcase2 = gr.Chatbot(
+                            [("hello", "world"), ("damn", "good")]
+                        )
+                    with gr.Tab("Ex3"):
+                        example_showcase3 = gr.Chatbot(
+                            [("hello", "world"), ("damn", "good")]
+                        )
+                    with gr.Tab("Ex4"):
+                        example_showcase4 = gr.Chatbot(
+                            [("hello", "world"), ("damn", "good")]
+                        )
+
+            with gr.Row():
+                back_to_model_choose_btn = gr.Button("Back")
+                confirm_btn = gr.Button("Confirm")
+
+            with gr.Column(elem_classes=["progress-view"]):
+                txt_view = gr.Textbox(label="Status")
+                progress_view2 = gr.Textbox(label="Progress")
+                
+    with gr.Column(visible=False) as chat_view:
+        idx = gr.State(0)
+        model_num = gr.State(0)
+        chat_state = gr.State()
+        local_data = gr.JSON({}, visible=False)
+
+        gr.Markdown("# Chatting", elem_classes=["center"])
+        gr.Markdown("""This entire application is built on top of `Gradio`. You can select one of the 10 channels on the left side to start chatting with the model. The model type you chose appear as a label on the top left corner of the chat component as well. Furthermore, you will see which model has responded to your question in each turn with their unique icons. This is because you can go back and forth to select different models from time to time, and you can continue your conversation with different models. With models' icons, you will understand how the conversation has gone better.""")
+        
+        with gr.Row():
+            with gr.Column(scale=1, min_width=180):
+                gr.Markdown("GradioChat", elem_id="left-top")
+
+                with gr.Column(elem_id="left-pane"):
+                    chat_back_btn = gr.Button("Back", elem_id="chat-back-btn")
+                    
+                    with gr.Accordion("Histories", elem_id="chat-history-accordion"):
+                        channel_btns.append(gr.Button(channels[0], elem_classes=["custom-btn-highlight"]))
+
+                        for channel in channels[1:]:
+                            channel_btns.append(gr.Button(channel, elem_classes=["custom-btn"]))
+
+            with gr.Column(scale=8, elem_id="right-pane"):
+                with gr.Column(
+                    elem_id="initial-popup", visible=False
+                ) as example_block:
+                    with gr.Row(scale=1):
+                        with gr.Column(elem_id="initial-popup-left-pane"):
+                            gr.Markdown("GradioChat", elem_id="initial-popup-title")
+                            gr.Markdown(
+                                "Making the community's best AI chat models available to everyone."
+                            )
+                        with gr.Column(elem_id="initial-popup-right-pane"):
+                            gr.Markdown(
+                                "Chat UI is now open sourced on Hugging Face Hub"
+                            )
+                            gr.Markdown(
+                                "check out the [↗ repository](https://huggingface.co/spaces/chansung/test-multi-conv)"
+                            )
+
+                    with gr.Column(scale=1):
+                        gr.Markdown("Examples")
+                        with gr.Row():
+                            for example in examples:
+                                ex_btns.append(gr.Button(example, elem_classes=["example-btn"]))
+
+                with gr.Column(elem_id="aux-btns-popup", visible=True):
+                    with gr.Row():
+                        stop = gr.Button("Stop", elem_classes=["aux-btn"], interactive=False)
+                        regenerate = gr.Button("Rege", interactive=False, elem_classes=["aux-btn"])
+                        clean = gr.Button("Clean", elem_classes=["aux-btn"])
+
+                with gr.Accordion("Context Inspector", elem_id="aux-viewer", open=False):
+                    context_inspector = gr.Textbox(
+                        "",
+                        elem_id="aux-viewer-inspector",
+                        label="",
+                        lines=30,
+                        max_lines=50,
+                    )                        
+                        
+                chatbot = gr.Chatbot(elem_id='chatbot')
+                instruction_txtbox = gr.Textbox(
+                    placeholder="Ask anything", label="",
+                    elem_id="prompt-txt"
+                )
+
+        with gr.Accordion("Control Panel", open=False) as control_panel:
+            with gr.Column():
+                with gr.Column():
+                    gr.Markdown("#### Global context")
+                    with gr.Accordion("global context will persist during conversation, and it is placed at the top of the prompt", open=False):
+                        global_context = gr.Textbox(
+                            "global context",
+                            lines=5,
+                            max_lines=10,
+                            interactive=True,
+                            elem_id="global-context"
+                        )
+
+                    gr.Markdown("#### GenConfig for **response** text generation")
+                    with gr.Row():
+                        res_temp = gr.Slider(0.0, 2.0, 0, step=0.1, label="temp", interactive=True)
+                        res_topp = gr.Slider(0.0, 2.0, 0, step=0.1, label="top_p", interactive=True)
+                        res_topk = gr.Slider(20, 1000, 0, step=1, label="top_k", interactive=True)
+                        res_rpen = gr.Slider(0.0, 2.0, 0, step=0.1, label="rep_penalty", interactive=True)
+                        res_mnts = gr.Slider(64, 2048, 0, step=1, label="new_tokens", interactive=True)                            
+                        res_beams = gr.Slider(1, 4, 0, step=1, label="beams")
+                        res_cache = gr.Radio([True, False], value=0, label="cache", interactive=True)
+                        res_sample = gr.Radio([True, False], value=0, label="sample", interactive=True)
+                        res_eosid = gr.Number(value=0, visible=False, precision=0)
+                        res_padid = gr.Number(value=0, visible=False, precision=0)
+
+                with gr.Column(visible=False):
+                    gr.Markdown("#### GenConfig for **summary** text generation")
+                    with gr.Row():
+                        sum_temp = gr.Slider(0.0, 2.0, 0, step=0.1, label="temp", interactive=True)
+                        sum_topp = gr.Slider(0.0, 2.0, 0, step=0.1, label="top_p", interactive=True)
+                        sum_topk = gr.Slider(20, 1000, 0, step=1, label="top_k", interactive=True)
+                        sum_rpen = gr.Slider(0.0, 2.0, 0, step=0.1, label="rep_penalty", interactive=True)
+                        sum_mnts = gr.Slider(64, 2048, 0, step=1, label="new_tokens", interactive=True)
+                        sum_beams = gr.Slider(1, 8, 0, step=1, label="beams", interactive=True)
+                        sum_cache = gr.Radio([True, False], value=0, label="cache", interactive=True)
+                        sum_sample = gr.Radio([True, False], value=0, label="sample", interactive=True)
+                        sum_eosid = gr.Number(value=0, visible=False, precision=0)
+                        sum_padid = gr.Number(value=0, visible=False, precision=0)
+
+                with gr.Column():
+                    gr.Markdown("#### Context managements")
+                    with gr.Row():
+                        ctx_num_lconv = gr.Slider(2, 10, 3, step=1, label="number of recent talks to keep", interactive=True)
+                        ctx_sum_prompt = gr.Textbox(
+                            "summarize our conversations. what have we discussed about so far?",
+                            label="design a prompt to summarize the conversations",
+                            visible=False
+                        )
+
+        gr.Markdown("""The control panel on the bottom side allows you to adjust three major hyper-parameters. First, you can set the global context of the conversation. Appropriate global context that is recommended by each model's authors is provided by default, but you can set it as you like. Second, you can adjust some of the hyper-parameters of the `GenerationConfig` to decide how you want the model to generate text. `Temperature`, `Top K`, and `New Max Tokens` are some of the available ones. Third, you can adjust the number of recent talks to keep track of. With bigger number, the model will see more of the past conversations.
+
+Lastly, there is a hidden panel on the top right corner, and it will appear when you hover your mouse around it. When expanding the panel, it shows what the model actually sees. That is you can double check how the entire prompt is constructed and fed into the model at each conversation.
+""")
+                        
+        btns = [
+            baize_7b, guanaco_7b #nous_hermes_13b, evolinstruct_vicuna_13b, guanaco_13b
+            # baize_7b, evolinstruct_vicuna_13b, guanaco_13b, nous_hermes_13b
+            # llama_deus_7b, koalpaca, evolinstruct_vicuna_13b, baize_7b, guanaco_33b,
+        ]
+        for btn in btns:
+            btn.click(
+                move_to_second_view,
+                btn,
+                [
+                    model_choice_view, model_review_view,
+                    model_image, model_name, model_params, model_base, model_ckpt,
+                    model_desc, model_vram, gen_config_path,
+                    example_showcase1, example_showcase2, example_showcase3, example_showcase4,
+                    progress_view
+                ]
+            )
+
+        back_to_model_choose_btn.click(
+            move_to_first_view,
+            None,
+            [model_choice_view, model_review_view, progress_view2]
+        )
+        
+        confirm_btn.click(
+            get_model_num,
+            [model_name],
+            [progress_view2, model_num]
+        ).then(
+            move_to_third_view,
+            model_num,
+            [progress_view2, model_review_view, chat_view, chatbot, chat_state, global_context,
+            res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid]
+        )
+         
+        for btn in channel_btns:
+            btn.click(
+                set_chatbot,
+                [btn, local_data, chat_state],
+                [chatbot, idx, example_block, regenerate]
+            ).then(
+                None, btn, None, 
+                _js=UPDATE_LEFT_BTNS_STATE        
+            )
+        
+        for btn in ex_btns:
+            btn.click(
+                set_example,
+                [btn],
+                [instruction_txtbox, example_block]  
+            )
+
+        instruction_txtbox.submit(
+            lambda: [
+                gr.update(visible=False),
+                gr.update(interactive=True)
+            ],
+            None,
+            [example_block, regenerate]
+        ).then(
+            central.chat_stream,
+            [idx, local_data, instruction_txtbox, chat_state, model_num,
+            global_context, ctx_num_lconv, ctx_sum_prompt,
+            res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid],
+            [instruction_txtbox, chatbot, context_inspector, local_data],
+        ).then(
+            None, local_data, None, 
+            _js="(v)=>{ setStorage('local_data',v) }"
+        )
+
+        regenerate.click(
+            rollback_last,
+            [idx, local_data, chat_state],
+            [instruction_txtbox, chatbot, local_data, regenerate]
+        ).then(
+            central.chat_stream,
+            [idx, local_data, instruction_txtbox, chat_state, model_num,
+            global_context, ctx_num_lconv, ctx_sum_prompt,
+            res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid],
+            [instruction_txtbox, chatbot, context_inspector, local_data],            
+        ).then(
+            lambda: gr.update(interactive=True),
+            None,
+            regenerate
+        ).then(
+            None, local_data, None, 
+            _js="(v)=>{ setStorage('local_data',v) }"  
+        )
+        
+        # stop.click(
+        #     None, None, None,
+        #     cancels=[send_event]
+        # )
+
+        clean.click(
+            reset_chat,
+            [idx, local_data, chat_state],
+            [instruction_txtbox, chatbot, local_data, example_block, regenerate]
+        ).then(
+            None, local_data, None, 
+            _js="(v)=>{ setStorage('local_data',v) }"
+        )
+        
+        chat_back_btn.click(
+            lambda: [gr.update(visible=False), gr.update(visible=True)],
+            None,
+            [chat_view, model_choice_view]
+        )
+        
+        demo.load(
+          None,
+          inputs=None,
+          outputs=[chatbot, local_data],
+          _js=GET_LOCAL_STORAGE,
+        )          
+        
+demo.queue(
+    concurrency_count=5,
+    max_size=256,
+).launch(
+    server_port=6006, 
+    server_name="0.0.0.0", 
+    debug=True,
+)

args.py

@@ -0,0 +1,61 @@
+import argparse
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="Gradio Application for LLM as a chatbot service"
+    )
+    parser.add_argument(
+        "--base-url",
+        help="Hugging Face Hub URL",
+        default="elinas/llama-7b-hf-transformers-4.29",
+        type=str,
+    )
+    parser.add_argument(
+        "--ft-ckpt-url",
+        help="Hugging Face Hub URL",
+        # default="tloen/alpaca-lora-7b",
+        default="LLMs/Alpaca-LoRA-7B-elina",
+        type=str,
+    )
+    parser.add_argument(
+        "--port",
+        help="PORT number where the app is served",
+        default=6006,
+        type=int,
+    )
+    parser.add_argument(
+        "--share",
+        help="Create and share temporary endpoint (useful in Colab env)",
+        action='store_true'
+    )
+    parser.add_argument(
+        "--gen-config-path",
+        help="path to GenerationConfig file",
+        default="configs/response_configs/default.yaml",
+        # default="configs/gen_config_koalpaca.yaml",
+        # default="configs/gen_config_stablelm.yaml",
+        type=str
+    )
+    parser.add_argument(
+        "--gen-config-summarization-path",
+        help="path to GenerationConfig file used in context summarization",
+        default="configs/summarization_configs/default.yaml",
+        type=str
+    )
+    parser.add_argument(
+        "--multi-gpu",
+        help="Enable multi gpu mode. This will force not to use Int8 but float16, so you need to check if your system has enough GPU memory",
+        action='store_true'
+    )
+    parser.add_argument(
+        "--force-download_ckpt",
+        help="Force to download ckpt instead of using cached one",
+        action="store_true"
+    )
+    parser.add_argument(
+        "--chat-only-mode",
+        help="Only show chatting window. Otherwise, other components will be appeared for more sophisticated control",
+        action="store_true"
+    )
+    
+    return parser.parse_args()

channels.txt

@@ -0,0 +1,10 @@
+1st Channel
+2nd Channel
+3rd Channel
+4th Channel
+5th Channel
+6th Channel
+7th Channel
+8th Channel
+9th Channel
+10th Channel

chats/alpaca.py

@@ -0,0 +1,108 @@
+import copy
+import json
+import global_vars
+from chats import pre, post
+from pingpong import PingPong
+from gens.batch_gen import get_output_batch
+
+from pingpong.context import CtxLastWindowStrategy
+
+def build_prompts(ppmanager, user_message, global_context, win_size=3):
+    dummy_ppm = copy.deepcopy(ppmanager)
+    
+    dummy_ppm.ctx = global_context
+    for pingpong in dummy_ppm.pingpongs:
+        pong = pingpong.pong
+        first_sentence = pong.split("\n")[0]
+        if first_sentence != "" and \
+            pre.contains_image_markdown(first_sentence):
+            pong = ' '.join(pong.split("\n")[1:]).strip()
+            pingpong.pong = pong
+            
+    lws = CtxLastWindowStrategy(win_size)
+    
+    prompt = lws(dummy_ppm)
+    return prompt
+
+def text_stream(ppmanager, streamer, model_thumbnail_tiny, model_type):
+    count = 0
+    
+    for new_text in streamer:
+        if count == 0:
+            ppmanager.append_pong(f"![]({model_thumbnail_tiny})***[{model_type}]***\n")
+            count = count + 1        
+        
+        ppmanager.append_pong(new_text)
+        yield ppmanager, ppmanager.build_uis()
+                
+    yield ppmanager, ppmanager.build_uis()
+
+def summarize(
+    ppmanager, prompt_to_summarize, win_size,
+    temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+    num_beams, use_cache, do_sample, eos_token_id, pad_token_id    
+):
+    ctx = ppmanager.ctx
+    last_pong = ppmanager.pingpongs[-1].pong
+    ppmanager.add_pingpong(PingPong(prompt_to_summarize, ""))
+    prompt = ppmanager.build_prompts(from_idx=-win_size)
+    
+    _, gen_config_summarization = pre.build_gen_config(
+        temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+        num_beams, use_cache, do_sample, eos_token_id, pad_token_id
+    )
+    summarize_output = get_output_batch(
+        global_vars.model, global_vars.tokenizer, [prompt], gen_config_summarization
+    )[0].split("### Response:")[-1].strip()
+    ppmanager.ctx = summarize_output
+    ppmanager.pop_pingpong()
+    return ppmanager
+
+def chat_stream(
+    idx, local_data, user_message, state, model_num,
+    global_context, ctx_num_lconv, ctx_sum_prompt,
+    res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+):
+    res = [
+      state["ppmanager_type"].from_json(json.dumps(ppm))
+      for ppm in local_data
+    ]
+
+    ppm = res[idx]
+
+    # add_ping returns a prompt structured in Alpaca form
+    ppm.add_pingpong(
+        PingPong(user_message, "")
+    )
+    prompt = build_prompts(ppm, user_message, global_context, ctx_num_lconv)
+
+    # prepare text generating streamer & start generating
+    gen_kwargs, streamer = pre.build(
+        prompt, model_num,
+        res_temp, res_topp, res_topk, res_rpen, res_mnts, 
+        res_beams, res_cache, res_sample, res_eosid, res_padid,
+        return_token_type_ids=False
+    )
+    pre.start_gen(gen_kwargs, model_num)
+
+    model_thumbnail_tiny = global_vars.models[model_num]["model_thumb_tiny"]
+    model_type = global_vars.models[model_num]["model_type"]
+    for ppmanager, uis in text_stream(ppm, streamer, model_thumbnail_tiny, model_type):
+        yield "", uis, prompt, str(res)
+
+    ppm = post.strip_pong(ppm)
+    yield "", ppm.build_uis(), prompt, str(res)
+    
+    # summarization
+    # ppm.add_pingpong(
+    #     PingPong(None, "![](https://i.postimg.cc/ZKNKDPBd/Vanilla-1s-209px.gif)")
+    # )
+    # yield "", ppm.build_uis(), prompt, state
+    # ppm.pop_pingpong()
+    
+    # ppm = summarize(
+    #     ppm, ctx_sum_prompt, ctx_num_lconv,
+    #     sum_temp, sum_topp, sum_topk, sum_rpen, sum_mnts, 
+    #     sum_beams, sum_cache, sum_sample, sum_eosid, sum_padid
+    # )
+    yield "", ppm.build_uis(), prompt, str(res)

chats/alpaca_gpt4.py

@@ -0,0 +1,101 @@
+import copy
+import json
+import global_vars
+from chats import pre, post
+from pingpong import PingPong
+from gens.batch_gen import get_output_batch
+
+from pingpong.context import CtxLastWindowStrategy
+
+def build_prompts(ppmanager, user_message, global_context, win_size=3):
+    dummy_ppm = copy.deepcopy(ppmanager)
+    
+    dummy_ppm.ctx = global_context
+    for pingpong in dummy_ppm.pingpongs:
+        pong = pingpong.pong
+        first_sentence = pong.split("\n")[0]
+        if first_sentence != "" and \
+            pre.contains_image_markdown(first_sentence):
+            pong = ' '.join(pong.split("\n")[1:]).strip()
+            pingpong.pong = pong
+            
+    lws = CtxLastWindowStrategy(win_size)
+    
+    prompt = lws(dummy_ppm)
+    return prompt
+
+def text_stream(ppmanager, streamer):
+    for new_text in streamer:
+        ppmanager.append_pong(new_text)
+        yield ppmanager, ppmanager.build_uis()
+                
+    yield ppmanager, ppmanager.build_uis()
+
+def summarize(
+    ppmanager, prompt_to_summarize, win_size,
+    temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+    num_beams, use_cache, do_sample, eos_token_id, pad_token_id    
+):
+    ctx = ppmanager.ctx
+    last_pong = ppmanager.pingpongs[-1].pong
+    ppmanager.add_pingpong(PingPong(prompt_to_summarize, ""))
+    prompt = ppmanager.build_prompts(from_idx=-win_size)
+    
+    _, gen_config_summarization = pre.build_gen_config(
+        temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+        num_beams, use_cache, do_sample, eos_token_id, pad_token_id
+    )
+    summarize_output = get_output_batch(
+        global_vars.model, global_vars.tokenizer, [prompt], gen_config_summarization
+    )[0].split("### Response:")[-1].strip()
+    ppmanager.ctx = summarize_output
+    ppmanager.pop_pingpong()
+    return ppmanager
+
+def chat_stream(
+    idx, local_data, user_message, state, model_num,
+    global_context, ctx_num_lconv, ctx_sum_prompt,
+    res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+):
+    res = [
+      state["ppmanager_type"].from_json(json.dumps(ppm))
+      for ppm in local_data
+    ]
+
+    ppm = res[idx]
+
+    # add_ping returns a prompt structured in Alpaca form
+    ppm.add_pingpong(
+        PingPong(user_message, "")
+    )
+    prompt = build_prompts(ppm, user_message, global_context, ctx_num_lconv)
+
+    # prepare text generating streamer & start generating
+    gen_kwargs, streamer = pre.build(
+        prompt,
+        res_temp, res_topp, res_topk, res_rpen, res_mnts, 
+        res_beams, res_cache, res_sample, res_eosid, res_padid,
+        return_token_type_ids=False
+    )
+    pre.start_gen(gen_kwargs)
+
+    # handling stream
+    for ppmanager, uis in text_stream(ppm, streamer):
+        yield "", uis, prompt, str(res)
+
+    ppm = post.strip_pong(ppm)
+    yield "", ppm.build_uis(), prompt, str(res)
+    
+    # summarization
+    # ppm.add_pingpong(
+    #     PingPong(None, "![](https://i.postimg.cc/ZKNKDPBd/Vanilla-1s-209px.gif)")
+    # )
+    # yield "", ppm.build_uis(), prompt, state
+    # ppm.pop_pingpong()
+    
+    # ppm = summarize(
+    #     ppm, ctx_sum_prompt, ctx_num_lconv,
+    #     sum_temp, sum_topp, sum_topk, sum_rpen, sum_mnts, 
+    #     sum_beams, sum_cache, sum_sample, sum_eosid, sum_padid
+    # )
+    yield "", ppm.build_uis(), prompt, str(res)

chats/alpacoom.py

@@ -0,0 +1,101 @@
+import copy
+import json
+import global_vars
+from chats import pre, post
+from pingpong import PingPong
+from gens.batch_gen import get_output_batch
+
+from pingpong.context import CtxLastWindowStrategy
+
+def build_prompts(ppmanager, user_message, global_context, win_size=3):
+    dummy_ppm = copy.deepcopy(ppmanager)
+    
+    dummy_ppm.ctx = global_context
+    for pingpong in dummy_ppm.pingpongs:
+        pong = pingpong.pong
+        first_sentence = pong.split("\n")[0]
+        if first_sentence != "" and \
+            pre.contains_image_markdown(first_sentence):
+            pong = ' '.join(pong.split("\n")[1:]).strip()
+            pingpong.pong = pong
+            
+    lws = CtxLastWindowStrategy(win_size)
+    
+    prompt = lws(dummy_ppm)
+    return prompt
+
+def text_stream(ppmanager, streamer):
+    for new_text in streamer:
+        ppmanager.append_pong(new_text)
+        yield ppmanager, ppmanager.build_uis()
+                
+    yield ppmanager, ppmanager.build_uis()
+
+def summarize(
+    ppmanager, prompt_to_summarize, win_size,
+    temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+    num_beams, use_cache, do_sample, eos_token_id, pad_token_id    
+):
+    ctx = ppmanager.ctx
+    last_pong = ppmanager.pingpongs[-1].pong
+    ppmanager.add_pingpong(PingPong(prompt_to_summarize, ""))
+    prompt = ppmanager.build_prompts(from_idx=-win_size)
+    
+    _, gen_config_summarization = pre.build_gen_config(
+        temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+        num_beams, use_cache, do_sample, eos_token_id, pad_token_id
+    )
+    summarize_output = get_output_batch(
+        global_vars.model, global_vars.tokenizer, [prompt], gen_config_summarization
+    )[0].split("### Response:")[-1].strip()
+    ppmanager.ctx = summarize_output
+    ppmanager.pop_pingpong()
+    return ppmanager
+
+def chat_stream(
+    idx, local_data, user_message, state, model_num,
+    global_context, ctx_num_lconv, ctx_sum_prompt,
+    res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+):
+    res = [
+      state["ppmanager_type"].from_json(json.dumps(ppm))
+      for ppm in local_data
+    ]
+
+    ppm = res[idx]
+
+    # add_ping returns a prompt structured in Alpaca form
+    ppm.add_pingpong(
+        PingPong(user_message, "")
+    )
+    prompt = build_prompts(ppm, user_message, global_context, ctx_num_lconv)
+
+    # prepare text generating streamer & start generating
+    gen_kwargs, streamer = pre.build(
+        prompt,
+        res_temp, res_topp, res_topk, res_rpen, res_mnts, 
+        res_beams, res_cache, res_sample, res_eosid, res_padid,
+        return_token_type_ids=False
+    )
+    pre.start_gen(gen_kwargs)
+
+    # handling stream
+    for ppmanager, uis in text_stream(ppm, streamer):
+        yield "", uis, prompt, str(res)
+
+    ppm = post.strip_pong(ppm)
+    yield "", ppm.build_uis(), prompt, str(res)
+    
+    # summarization
+    # ppm.add_pingpong(
+    #     PingPong(None, "![](https://i.postimg.cc/ZKNKDPBd/Vanilla-1s-209px.gif)")
+    # )
+    # yield "", ppm.build_uis(), prompt, state
+    # ppm.pop_pingpong()
+    
+    # ppm = summarize(
+    #     ppm, ctx_sum_prompt, ctx_num_lconv,
+    #     sum_temp, sum_topp, sum_topk, sum_rpen, sum_mnts, 
+    #     sum_beams, sum_cache, sum_sample, sum_eosid, sum_padid
+    # )
+    yield "", ppm.build_uis(), prompt, str(res)

chats/baize.py

@@ -0,0 +1,113 @@
+import copy
+import json
+import global_vars
+from chats import pre, post
+from pingpong import PingPong
+from gens.batch_gen import get_output_batch
+
+from pingpong.context import CtxLastWindowStrategy
+
+def build_prompts(ppmanager, user_message, global_context, win_size=3):
+    dummy_ppm = copy.deepcopy(ppmanager)
+    
+    dummy_ppm.ctx = global_context
+    for pingpong in dummy_ppm.pingpongs:
+        pong = pingpong.pong
+        first_sentence = pong.split("\n")[0]
+        if first_sentence != "" and \
+            pre.contains_image_markdown(first_sentence):
+            pong = ' '.join(pong.split("\n")[1:]).strip()
+            pingpong.pong = pong
+            
+    lws = CtxLastWindowStrategy(win_size)
+    
+    prompt = lws(dummy_ppm)
+    return prompt
+
+def text_stream(ppmanager, streamer, model_thumbnail_tiny, model_type):
+    count = 0
+    
+    for new_text in streamer:
+        if "[|Human|]" in new_text or \
+            "[|AI|]" in new_text:
+            break
+            
+        if count == 0:
+            ppmanager.append_pong(f"![]({model_thumbnail_tiny})***[{model_type}]***\n")
+            count = count + 1
+            
+        ppmanager.append_pong(new_text)
+        yield ppmanager, ppmanager.build_uis()
+                
+    yield ppmanager, ppmanager.build_uis()
+
+def summarize(
+    ppmanager, prompt_to_summarize, win_size,
+    temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+    num_beams, use_cache, do_sample, eos_token_id, pad_token_id    
+):
+    ctx = ppmanager.ctx
+    last_pong = ppmanager.pingpongs[-1].pong
+    ppmanager.add_pingpong(PingPong(prompt_to_summarize, ""))
+    prompt = ppmanager.build_prompts(from_idx=-win_size)
+    
+    _, gen_config_summarization = pre.build_gen_config(
+        temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+        num_beams, use_cache, do_sample, eos_token_id, pad_token_id
+    )
+    summarize_output = get_output_batch(
+        global_vars.model, global_vars.tokenizer, [prompt], gen_config_summarization
+    )[0].split("### Response:")[-1].strip()
+    ppmanager.ctx = summarize_output
+    ppmanager.pop_pingpong()
+    return ppmanager
+
+def chat_stream(
+    idx, local_data, user_message, state, model_num,
+    global_context, ctx_num_lconv, ctx_sum_prompt,
+    res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+):
+    res = [
+      state["ppmanager_type"].from_json(json.dumps(ppm))
+      for ppm in local_data
+    ]
+
+    ppm = res[idx]
+
+    # add_ping returns a prompt structured in Alpaca form
+    ppm.add_pingpong(
+        PingPong(user_message, "")
+    )
+    prompt = build_prompts(ppm, user_message, global_context, ctx_num_lconv)
+
+    # prepare text generating streamer & start generating
+    gen_kwargs, streamer = pre.build(
+        prompt, model_num,
+        res_temp, res_topp, res_topk, res_rpen, res_mnts, 
+        res_beams, res_cache, res_sample, res_eosid, res_padid,
+        return_token_type_ids=False
+    )
+    pre.start_gen(gen_kwargs, model_num)
+
+    # handling stream
+    model_thumbnail_tiny = global_vars.models[model_num]["model_thumb_tiny"]
+    model_type = global_vars.models[model_num]["model_type"]
+    for ppmanager, uis in text_stream(ppm, streamer, model_thumbnail_tiny, model_type):
+        yield "", uis, prompt, str(res)
+
+    ppm = post.strip_pong(ppm)
+    yield "", ppm.build_uis(), prompt, str(res)
+    
+    # summarization
+    # ppm.add_pingpong(
+    #     PingPong(None, "![](https://i.postimg.cc/ZKNKDPBd/Vanilla-1s-209px.gif)")
+    # )
+    # yield "", ppm.build_uis(), prompt, state
+    # ppm.pop_pingpong()
+    
+    # ppm = summarize(
+    #     ppm, ctx_sum_prompt, ctx_num_lconv,
+    #     sum_temp, sum_topp, sum_topk, sum_rpen, sum_mnts, 
+    #     sum_beams, sum_cache, sum_sample, sum_eosid, sum_padid
+    # )
+    yield "", ppm.build_uis(), prompt, str(res)

chats/central.py

@@ -0,0 +1,156 @@
+from chats import stablelm
+from chats import alpaca
+from chats import koalpaca
+from chats import flan_alpaca
+from chats import os_stablelm
+from chats import vicuna
+from chats import starchat
+from chats import redpajama
+from chats import mpt
+from chats import alpacoom
+from chats import baize
+from chats import guanaco
+
+def chat_stream(
+    idx, local_data, user_message, state, model_num,
+    global_context, ctx_num_lconv, ctx_sum_prompt,
+    res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+):
+    model_type = state["model_type"]
+
+    if model_type == "stablelm":
+        cs = stablelm.chat_stream(
+            idx, local_data, user_message, state, model_num,
+            global_context, ctx_num_lconv, ctx_sum_prompt,
+            res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+        )
+
+    elif model_type == "baize":
+        cs = baize.chat_stream(
+            idx, local_data, user_message, state, model_num,
+            global_context, ctx_num_lconv, ctx_sum_prompt,
+            res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+        )        
+        
+    elif model_type == "alpaca":
+        cs = alpaca.chat_stream(
+            idx, local_data, user_message, state, model_num,
+            global_context, ctx_num_lconv, ctx_sum_prompt,
+            res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+        )
+
+    elif model_type == "alpaca-gpt4":
+        cs = alpaca.chat_stream(
+            idx, local_data, user_message, state, model_num,
+            global_context, ctx_num_lconv, ctx_sum_prompt,
+            res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+        )
+
+    elif model_type == "alpacoom":
+        cs = alpacoom.chat_stream(
+            idx, local_data, user_message, state, model_num,
+            global_context, ctx_num_lconv, ctx_sum_prompt,
+            res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+        )        
+
+    elif model_type == "llama-deus":
+        cs = alpaca.chat_stream(
+            idx, local_data, user_message, state, model_num,
+            global_context, ctx_num_lconv, ctx_sum_prompt,
+            res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+        )
+
+    elif model_type == "camel":
+        cs = alpaca.chat_stream(
+            idx, local_data, user_message, state, model_num,
+            global_context, ctx_num_lconv, ctx_sum_prompt,
+            res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+        )
+
+    elif model_type == "koalpaca-polyglot":
+        cs = koalpaca.chat_stream(
+            idx, local_data, user_message, state, model_num,
+            global_context, ctx_num_lconv, ctx_sum_prompt,
+            res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+        )
+
+    elif model_type == "flan-alpaca":
+        cs = flan_alpaca.chat_stream(
+            idx, local_data, user_message, state, model_num,
+            global_context, ctx_num_lconv, ctx_sum_prompt,
+            res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+        )        
+
+    elif model_type == "os-stablelm":
+        cs = os_stablelm.chat_stream(
+            idx, local_data, user_message, state, model_num,
+            global_context, ctx_num_lconv, ctx_sum_prompt,
+            res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+        )
+
+    elif model_type == "t5-vicuna":
+        cs = vicuna.chat_stream(
+            idx, local_data, user_message, state, model_num,
+            global_context, ctx_num_lconv, ctx_sum_prompt,
+            res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+        )
+
+    elif model_type == "stable-vicuna":
+        cs = vicuna.chat_stream(
+            idx, local_data, user_message, state, model_num,
+            global_context, ctx_num_lconv, ctx_sum_prompt,
+            res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+        )
+
+    elif model_type == "vicuna":
+        cs = vicuna.chat_stream(
+            idx, local_data, user_message, state, model_num,
+            global_context, ctx_num_lconv, ctx_sum_prompt,
+            res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+        )        
+
+    elif model_type == "evolinstruct-vicuna":
+        cs = vicuna.chat_stream(
+            idx, local_data, user_message, state, model_num,
+            global_context, ctx_num_lconv, ctx_sum_prompt,
+            res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+        )
+
+    elif model_type == "starchat":
+        cs = starchat.chat_stream(
+            idx, local_data, user_message, state, model_num,
+            global_context, ctx_num_lconv, ctx_sum_prompt,
+            res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+        )
+
+    elif model_type == "mpt":
+        cs = mpt.chat_stream(
+            idx, local_data, user_message, state, model_num,
+            global_context, ctx_num_lconv, ctx_sum_prompt,
+            res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+        )
+
+    elif model_type == "redpajama":
+        cs = redpajama.chat_stream(
+            idx, local_data, user_message, state, model_num,
+            global_context, ctx_num_lconv, ctx_sum_prompt,
+            res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+        )
+
+    elif model_type == "guanaco":
+        cs = guanaco.chat_stream(
+            idx, local_data, user_message, state, model_num,
+            global_context, ctx_num_lconv, ctx_sum_prompt,
+            res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+        )
+
+    elif model_type == "nous-hermes":
+        cs = alpaca.chat_stream(
+            idx, local_data, user_message, state, model_num,
+            global_context, ctx_num_lconv, ctx_sum_prompt,
+            res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+        )        
+        
+    for idx, x in enumerate(cs):
+        yield x        
+        

chats/flan_alpaca.py

@@ -0,0 +1,101 @@
+import copy
+import json
+import global_vars
+from chats import pre, post
+from pingpong import PingPong
+from gens.batch_gen import get_output_batch
+
+from pingpong.context import CtxLastWindowStrategy
+
+def build_prompts(ppmanager, user_message, global_context, win_size=3):
+    dummy_ppm = copy.deepcopy(ppmanager)
+    
+    dummy_ppm.ctx = global_context
+    for pingpong in dummy_ppm.pingpongs:
+        pong = pingpong.pong
+        first_sentence = pong.split("\n")[0]
+        if first_sentence != "" and \
+            pre.contains_image_markdown(first_sentence):
+            pong = ' '.join(pong.split("\n")[1:]).strip()
+            pingpong.pong = pong
+            
+    lws = CtxLastWindowStrategy(win_size)
+    
+    prompt = lws(dummy_ppm)
+    return prompt
+
+def text_stream(ppmanager, streamer):
+    for new_text in streamer:
+        ppmanager.append_pong(new_text)
+        yield ppmanager, ppmanager.build_uis()
+                
+    yield ppmanager, ppmanager.build_uis()
+
+def summarize(
+    ppmanager, prompt_to_summarize, win_size,
+    temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+    num_beams, use_cache, do_sample, eos_token_id, pad_token_id    
+):
+    ctx = ppmanager.ctx
+    last_pong = ppmanager.pingpongs[-1].pong
+    ppmanager.add_pingpong(PingPong(prompt_to_summarize, ""))
+    prompt = ppmanager.build_prompts(from_idx=-win_size)
+    
+    _, gen_config_summarization = pre.build_gen_config(
+        temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+        num_beams, use_cache, do_sample, eos_token_id, pad_token_id
+    )
+    summarize_output = get_output_batch(
+        global_vars.model, global_vars.tokenizer, [prompt], gen_config_summarization
+    )[0].split("-----")[-1].strip()
+    ppmanager.ctx = summarize_output
+    ppmanager.pop_pingpong()
+    return ppmanager
+
+def chat_stream(
+    idx, local_data, user_message, state, model_num,
+    global_context, ctx_num_lconv, ctx_sum_prompt,
+    res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+):
+    res = [
+      state["ppmanager_type"].from_json(json.dumps(ppm))
+      for ppm in local_data
+    ]
+
+    ppm = res[idx]
+
+    # add_ping returns a prompt structured in Alpaca form
+    ppm.add_pingpong(
+        PingPong(user_message, "")
+    )
+    prompt = build_prompts(ppm, user_message, global_context, ctx_num_lconv)
+
+    # prepare text generating streamer & start generating
+    gen_kwargs, streamer = pre.build(
+        prompt,
+        res_temp, res_topp, res_topk, res_rpen, res_mnts, 
+        res_beams, res_cache, res_sample, res_eosid, res_padid,
+        return_token_type_ids=False
+    )
+    pre.start_gen(gen_kwargs)
+
+    # handling stream
+    for ppmanager, uis in text_stream(ppm, streamer):
+        yield "", uis, prompt, str(res)
+
+    ppm = post.strip_pong(ppm)
+    yield "", ppm.build_uis(), prompt, str(res)
+    
+    # summarization
+    # ppm.add_pingpong(
+    #     PingPong(None, "![](https://i.postimg.cc/ZKNKDPBd/Vanilla-1s-209px.gif)")
+    # )
+    # yield "", ppm.build_uis(), prompt, state
+    # ppm.pop_pingpong()
+    
+    # ppm = summarize(
+    #     ppm, ctx_sum_prompt, ctx_num_lconv,
+    #     sum_temp, sum_topp, sum_topk, sum_rpen, sum_mnts, 
+    #     sum_beams, sum_cache, sum_sample, sum_eosid, sum_padid
+    # )
+    yield "", ppm.build_uis(), prompt, str(res)

chats/guanaco.py

@@ -0,0 +1,120 @@
+import torch
+from transformers import StoppingCriteria, StoppingCriteriaList
+
+import copy
+import json
+import global_vars
+from chats import pre, post
+from pingpong import PingPong
+from gens.batch_gen import get_output_batch
+
+from pingpong.context import CtxLastWindowStrategy
+
+class StopOnTokens(StoppingCriteria):
+    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
+        stop_token_ids = [0]
+        
+        for stop_id in stop_token_ids:
+            if input_ids[0][-1] == stop_id:
+                return True
+        return False
+
+def build_prompts(ppmanager, user_message, global_context, win_size=3):
+    dummy_ppm = copy.deepcopy(ppmanager)
+    
+    dummy_ppm.ctx = global_context
+    for pingpong in dummy_ppm.pingpongs:
+        pong = pingpong.pong
+        first_sentence = pong.split("\n")[0]
+        if first_sentence != "" and \
+            pre.contains_image_markdown(first_sentence):
+            pong = ' '.join(pong.split("\n")[1:]).strip()
+            pingpong.pong = pong
+            
+    lws = CtxLastWindowStrategy(win_size)
+    
+    prompt = lws(dummy_ppm)
+    return prompt
+
+def text_stream(ppmanager, streamer, model_thumbnail_tiny, model_type):
+    count = 0
+    
+    for new_text in streamer:
+        if count == 0:
+            ppmanager.append_pong(f"![]({model_thumbnail_tiny})***[{model_type}]***\n")
+            count = count + 1        
+        
+        ppmanager.append_pong(new_text)
+        yield ppmanager, ppmanager.build_uis()
+                
+    yield ppmanager, ppmanager.build_uis()
+
+def summarize(
+    ppmanager, prompt_to_summarize, win_size,
+    temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+    num_beams, use_cache, do_sample, eos_token_id, pad_token_id    
+):
+    ctx = ppmanager.ctx
+    last_pong = ppmanager.pingpongs[-1].pong
+    ppmanager.add_pingpong(PingPong(prompt_to_summarize, ""))
+    prompt = ppmanager.build_prompts(from_idx=-win_size)
+
+    _, gen_config_summarization = pre.build_gen_config(
+        temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+        num_beams, use_cache, do_sample, eos_token_id, pad_token_id
+    )
+    summarize_output = get_output_batch(
+        global_vars.model, global_vars.tokenizer, [prompt], gen_config_summarization
+    )[0].split(prompt_to_summarize)[-1].strip()
+    ppmanager.ctx = summarize_output
+    ppmanager.pop_pingpong()
+    return ppmanager
+
+def chat_stream(
+    idx, local_data, user_message, state, model_num,
+    global_context, ctx_num_lconv, ctx_sum_prompt,
+    res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+):
+    res = [
+      state["ppmanager_type"].from_json(json.dumps(ppm))
+      for ppm in local_data
+    ]
+
+    ppm = res[idx]
+
+    # add_ping returns a prompt structured in Alpaca form
+    ppm.add_pingpong(
+        PingPong(user_message, "")
+    )
+    prompt = build_prompts(ppm, user_message, global_context, ctx_num_lconv)
+
+    # prepare text generating streamer & start generating
+    gen_kwargs, streamer = pre.build(
+        prompt, model_num,
+        res_temp, res_topp, res_topk, res_rpen, res_mnts, 
+        res_beams, res_cache, res_sample, res_eosid, res_padid,
+        StoppingCriteriaList([StopOnTokens()]), False
+    )
+    pre.start_gen(gen_kwargs, model_num)
+
+    model_thumbnail_tiny = global_vars.models[model_num]["model_thumb_tiny"]
+    model_type = global_vars.models[model_num]["model_type"]
+    for ppmanager, uis in text_stream(ppm, streamer, model_thumbnail_tiny, model_type):
+        yield "", uis, prompt, str(res)
+
+    ppm = post.strip_pong(ppm)
+    yield "", ppm.build_uis(), prompt, str(res)
+    
+    # summarization
+    # ppm.add_pingpong(
+    #     PingPong(None, "![](https://i.postimg.cc/ZKNKDPBd/Vanilla-1s-209px.gif)")
+    # )
+    # yield "", ppm.build_uis(), prompt, state
+    # ppm.pop_pingpong()
+    
+    # ppm = summarize(
+    #     ppm, ctx_sum_prompt, ctx_num_lconv,
+    #     sum_temp, sum_topp, sum_topk, sum_rpen, sum_mnts, 
+    #     sum_beams, sum_cache, sum_sample, sum_eosid, sum_padid
+    # )
+    yield "", ppm.build_uis(), prompt, str(res)    

chats/koalpaca.py

@@ -0,0 +1,101 @@
+import copy
+import json
+import global_vars
+from chats import pre, post
+from pingpong import PingPong
+from gens.batch_gen import get_output_batch
+
+from pingpong.context import CtxLastWindowStrategy
+
+def build_prompts(ppmanager, user_message, global_context, win_size=3):
+    dummy_ppm = copy.deepcopy(ppmanager)
+    
+    dummy_ppm.ctx = global_context
+    for pingpong in dummy_ppm.pingpongs:
+        pong = pingpong.pong
+        first_sentence = pong.split("\n")[0]
+        if first_sentence != "" and \
+            pre.contains_image_markdown(first_sentence):
+            pong = ' '.join(pong.split("\n")[1:]).strip()
+            pingpong.pong = pong
+            
+    lws = CtxLastWindowStrategy(win_size)
+    
+    prompt = lws(dummy_ppm)
+    return prompt
+
+def text_stream(ppmanager, streamer):
+    for new_text in streamer:
+        ppmanager.append_pong(new_text)
+        yield ppmanager, ppmanager.build_uis()
+                
+    yield ppmanager, ppmanager.build_uis()
+
+def summarize(
+    ppmanager, prompt_to_summarize, win_size,
+    temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+    num_beams, use_cache, do_sample, eos_token_id, pad_token_id    
+):
+    ctx = ppmanager.ctx
+    last_pong = ppmanager.pingpongs[-1].pong
+    ppmanager.add_pingpong(PingPong(prompt_to_summarize, ""))
+    prompt = ppmanager.build_prompts(from_idx=-win_size)
+    
+    _, gen_config_summarization = pre.build_gen_config(
+        temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+        num_beams, use_cache, do_sample, eos_token_id, pad_token_id
+    )
+    summarize_output = get_output_batch(
+        global_vars.model, global_vars.tokenizer, [prompt], gen_config_summarization
+    )[0].split("### 응답:")[-1].strip()
+    ppmanager.ctx = summarize_output
+    ppmanager.pop_pingpong()
+    return ppmanager
+
+def chat_stream(
+    idx, local_data, user_message, state, model_num,
+    global_context, ctx_num_lconv, ctx_sum_prompt,
+    res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+):
+    res = [
+      state["ppmanager_type"].from_json(json.dumps(ppm))
+      for ppm in local_data
+    ]
+
+    ppm = res[idx]
+
+    # add_ping returns a prompt structured in Alpaca form
+    ppm.add_pingpong(
+        PingPong(user_message, "")
+    )
+    prompt = build_prompts(ppm, user_message, global_context, ctx_num_lconv)
+
+    # prepare text generating streamer & start generating
+    gen_kwargs, streamer = pre.build(
+        prompt, model_num,
+        res_temp, res_topp, res_topk, res_rpen, res_mnts, 
+        res_beams, res_cache, res_sample, res_eosid, res_padid,
+        return_token_type_ids=False
+    )
+    pre.start_gen(gen_kwargs, model_num)
+
+    # handling stream
+    for ppmanager, uis in text_stream(ppm, streamer):
+        yield "", uis, prompt, str(res)
+
+    ppm = post.strip_pong(ppm)
+    yield "", ppm.build_uis(), prompt, str(res)
+    
+    # summarization
+    # ppm.add_pingpong(
+    #     PingPong(None, "![](https://i.postimg.cc/ZKNKDPBd/Vanilla-1s-209px.gif)")
+    # )
+    # yield "", ppm.build_uis(), prompt, state
+    # ppm.pop_pingpong()
+    
+    # ppm = summarize(
+    #     ppm, ctx_sum_prompt, ctx_num_lconv,
+    #     sum_temp, sum_topp, sum_topk, sum_rpen, sum_mnts, 
+    #     sum_beams, sum_cache, sum_sample, sum_eosid, sum_padid
+    # )
+    yield "", ppm.build_uis(), prompt, str(res)

chats/mpt.py

@@ -0,0 +1,118 @@
+import torch
+from transformers import StoppingCriteria, StoppingCriteriaList
+
+import copy
+import json
+import global_vars
+from chats import pre, post
+from pingpong import PingPong
+from gens.batch_gen import get_output_batch
+
+from pingpong.context import CtxLastWindowStrategy
+
+class StopOnTokens(StoppingCriteria):
+    def __init__(self, tokenizer):
+        super().__init__()
+
+        self.stop_token_ids = tokenizer.convert_tokens_to_ids(
+            ["<|im_end|>", "<|endoftext|>"]
+        )
+
+    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
+        for stop_id in self.stop_token_ids:
+            if input_ids[0][-1] == stop_id:
+                return True
+        return False
+
+def build_prompts(ppmanager, user_message, global_context, win_size=3):
+    dummy_ppm = copy.deepcopy(ppmanager)
+    
+    dummy_ppm.ctx = global_context
+    for pingpong in dummy_ppm.pingpongs:
+        pong = pingpong.pong
+        first_sentence = pong.split("\n")[0]
+        if first_sentence != "" and \
+            pre.contains_image_markdown(first_sentence):
+            pong = ' '.join(pong.split("\n")[1:]).strip()
+            pingpong.pong = pong
+            
+    lws = CtxLastWindowStrategy(win_size)
+    
+    prompt = lws(dummy_ppm)
+    return prompt
+
+def text_stream(ppmanager, streamer):
+    for new_text in streamer:
+        ppmanager.append_pong(new_text)
+        yield ppmanager, ppmanager.build_uis()
+                
+    yield ppmanager, ppmanager.build_uis()
+
+def summarize(
+    ppmanager, prompt_to_summarize, win_size,
+    temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+    num_beams, use_cache, do_sample, eos_token_id, pad_token_id    
+):
+    ctx = ppmanager.ctx
+    last_pong = ppmanager.pingpongs[-1].pong
+    ppmanager.add_pingpong(PingPong(prompt_to_summarize, ""))
+    prompt = ppmanager.build_prompts(from_idx=-win_size)
+    
+    _, gen_config_summarization = pre.build_gen_config(
+        temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+        num_beams, use_cache, do_sample, eos_token_id, pad_token_id
+    )
+    summarize_output = get_output_batch(
+        global_vars.model, global_vars.tokenizer, [prompt], gen_config_summarization
+    )[0].strip()
+    ppmanager.ctx = summarize_output
+    ppmanager.pop_pingpong()
+    return ppmanager
+    
+def chat_stream(
+    idx, local_data, user_message, state, model_num,
+    global_context, ctx_num_lconv, ctx_sum_prompt,
+    res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+):
+    res = [
+      state["ppmanager_type"].from_json(json.dumps(ppm))
+      for ppm in local_data
+    ]
+
+    ppm = res[idx]
+
+    # add_ping returns a prompt structured in Alpaca form
+    ppm.add_pingpong(
+        PingPong(user_message, "")
+    )
+    prompt = build_prompts(ppm, user_message, global_context, ctx_num_lconv)
+
+    # prepare text generating streamer & start generating
+    gen_kwargs, streamer = pre.build(
+        prompt,
+        res_temp, res_topp, res_topk, res_rpen, res_mnts, 
+        res_beams, res_cache, res_sample, res_eosid, res_padid,
+        StoppingCriteriaList([StopOnTokens(global_vars.tokenizer)]), False
+    )
+    pre.start_gen(gen_kwargs)
+
+    # handling stream
+    for ppmanager, uis in text_stream(ppm, streamer):
+        yield "", uis, prompt, str(res)
+
+    ppm = post.strip_pong(ppm)
+    yield "", ppm.build_uis(), prompt, str(res)
+    
+    # summarization
+    # ppm.add_pingpong(
+    #     PingPong(None, "![](https://i.postimg.cc/ZKNKDPBd/Vanilla-1s-209px.gif)")
+    # )
+    # yield "", ppm.build_uis(), prompt, state
+    # ppm.pop_pingpong()
+    
+    # ppm = summarize(
+    #     ppm, ctx_sum_prompt, ctx_num_lconv,
+    #     sum_temp, sum_topp, sum_topk, sum_rpen, sum_mnts, 
+    #     sum_beams, sum_cache, sum_sample, sum_eosid, sum_padid
+    # )
+    yield "", ppm.build_uis(), prompt, str(res)    

chats/os_stablelm.py

@@ -0,0 +1,112 @@
+import torch
+from transformers import StoppingCriteria, StoppingCriteriaList
+
+import copy
+import json
+import global_vars
+from chats import pre, post
+from pingpong import PingPong
+from gens.batch_gen import get_output_batch
+
+from pingpong.context import CtxLastWindowStrategy
+
+class StopOnTokens(StoppingCriteria):
+    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
+        stop_ids = [50278, 50279, 50277, 1, 0]
+        for stop_id in stop_ids:
+            if input_ids[0][-1] == stop_id:
+                return True
+        return False
+
+def build_prompts(ppmanager, user_message, global_context, win_size=3):
+    dummy_ppm = copy.deepcopy(ppmanager)
+    
+    dummy_ppm.ctx = global_context
+    for pingpong in dummy_ppm.pingpongs:
+        pong = pingpong.pong
+        first_sentence = pong.split("\n")[0]
+        if first_sentence != "" and \
+            pre.contains_image_markdown(first_sentence):
+            pong = ' '.join(pong.split("\n")[1:]).strip()
+            pingpong.pong = pong
+            
+    lws = CtxLastWindowStrategy(win_size)
+    
+    prompt = lws(dummy_ppm)
+    return prompt
+
+def text_stream(ppmanager, streamer):
+    for new_text in streamer:
+        ppmanager.append_pong(new_text)
+        yield ppmanager, ppmanager.build_uis()
+                
+    yield ppmanager, ppmanager.build_uis()
+
+def summarize(
+    ppmanager, prompt_to_summarize, win_size,
+    temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+    num_beams, use_cache, do_sample, eos_token_id, pad_token_id    
+):
+    ctx = ppmanager.ctx
+    last_pong = ppmanager.pingpongs[-1].pong
+    ppmanager.add_pingpong(PingPong(prompt_to_summarize, ""))
+    prompt = ppmanager.build_prompts(from_idx=-win_size)
+
+    _, gen_config_summarization = pre.build_gen_config(
+        temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+        num_beams, use_cache, do_sample, eos_token_id, pad_token_id
+    )
+    summarize_output = get_output_batch(
+        global_vars.model, global_vars.tokenizer, [prompt], gen_config_summarization
+    )[0].split(prompt_to_summarize)[-1].strip()
+    ppmanager.ctx = summarize_output
+    ppmanager.pop_pingpong()
+    return ppmanager
+
+def chat_stream(
+    idx, local_data, user_message, state, model_num,
+    global_context, ctx_num_lconv, ctx_sum_prompt,
+    res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+):
+    res = [
+      state["ppmanager_type"].from_json(json.dumps(ppm))
+      for ppm in local_data
+    ]
+
+    ppm = res[idx]
+
+    # add_ping returns a prompt structured in Alpaca form
+    ppm.add_pingpong(
+        PingPong(user_message, "")
+    )
+    prompt = build_prompts(ppm, user_message, global_context, ctx_num_lconv)
+
+    # prepare text generating streamer & start generating
+    gen_kwargs, streamer = pre.build(
+        prompt,
+        res_temp, res_topp, res_topk, res_rpen, res_mnts, 
+        res_beams, res_cache, res_sample, res_eosid, res_padid,
+        StoppingCriteriaList([StopOnTokens()]), False
+    )
+    pre.start_gen(gen_kwargs)
+
+    # handling stream
+    for ppmanager, uis in text_stream(ppm, streamer):
+        yield "", uis, prompt, str(res)
+
+    ppm = post.strip_pong(ppm)
+    yield "", ppm.build_uis(), prompt, str(res)
+    
+    # summarization
+    # ppm.add_pingpong(
+    #     PingPong(None, "![](https://i.postimg.cc/ZKNKDPBd/Vanilla-1s-209px.gif)")
+    # )
+    # yield "", ppm.build_uis(), prompt, state
+    # ppm.pop_pingpong()
+    
+    # ppm = summarize(
+    #     ppm, ctx_sum_prompt, ctx_num_lconv,
+    #     sum_temp, sum_topp, sum_topk, sum_rpen, sum_mnts, 
+    #     sum_beams, sum_cache, sum_sample, sum_eosid, sum_padid
+    # )
+    yield "", ppm.build_uis(), prompt, str(res)    

chats/post.py

@@ -0,0 +1,3 @@
+def strip_pong(ppmanager):
+    ppmanager.pingpongs[-1].pong = ppmanager.pingpongs[-1].pong.strip()
+    return ppmanager

chats/pre.py

@@ -0,0 +1,97 @@
+import re
+import copy
+import global_vars
+from threading import Thread
+from transformers import TextIteratorStreamer
+from transformers import GenerationConfig
+
+def contains_image_markdown(string):
+    regex = re.compile(r'!\[(.*?)\]\((.*?)\)')
+    match = regex.search(string)
+    return match
+
+def build_model_inputs(prompt, model_num, return_token_type_ids):
+    model_inputs = global_vars.models[model_num]["tokenizer"](
+        [prompt], 
+        return_tensors="pt",
+        return_token_type_ids=return_token_type_ids
+    ).to("cuda")
+    return model_inputs
+
+def build_streamer(
+    model_num,
+    timeout=20.,
+    skip_prompt=True,
+    skip_special_tokens=True
+):
+    streamer = TextIteratorStreamer(
+        global_vars.models[model_num]["tokenizer"], 
+        timeout=timeout, 
+        skip_prompt=skip_prompt,
+        skip_special_tokens=skip_special_tokens
+    )
+    return streamer
+
+
+def build_gen_config(
+    temperature, top_p, top_k, repetition_penalty, max_new_tokens, 
+    num_beams, use_cache, do_sample, eos_token_id, pad_token_id 
+):
+    gen_config_raw = {
+        "temperature": temperature,
+        "top_p": top_p,
+        "top_k": top_k,
+        "repetition_penalty": repetition_penalty,
+        "max_new_tokens": max_new_tokens,
+        "num_beams": num_beams,
+        "use_cache": use_cache,
+        "do_sample": do_sample,
+        "eos_token_id": eos_token_id, 
+        "pad_token_id": pad_token_id
+    }
+
+    return gen_config_raw, GenerationConfig(**gen_config_raw)
+
+def build_gen_kwargs(
+    gen_config,
+    model_inputs,
+    streamer,
+    stopping_criteria
+):
+    gen_kwargs = dict(
+        model_inputs,
+        streamer=streamer,
+        stopping_criteria=stopping_criteria
+    )
+    gen_kwargs.update(gen_config)
+    return gen_kwargs 
+
+def start_gen(gen_kwargs, model_num):
+    t = Thread(
+        target=global_vars.models[model_num]["model"].generate,
+        kwargs=gen_kwargs
+    )
+    t.start()
+
+def build(
+    prompt, model_num,
+    temperature, top_p, top_k, repetition_penalty, max_new_tokens, 
+    num_beams, use_cache, do_sample, eos_token_id, pad_token_id,
+    stopping_criteria=None, return_token_type_ids=True
+):
+    gen_config_raw, _ = build_gen_config(
+        temperature, top_p, top_k, repetition_penalty, max_new_tokens, 
+        num_beams, use_cache, do_sample, eos_token_id, pad_token_id 
+    )
+
+    model_inputs = build_model_inputs(
+        prompt, model_num, return_token_type_ids=return_token_type_ids
+    )
+    streamer = build_streamer(model_num)
+    gen_kwargs = build_gen_kwargs(
+        gen_config_raw, 
+        model_inputs, 
+        streamer,
+        stopping_criteria
+    )
+    return gen_kwargs, streamer

chats/redpajama.py

@@ -0,0 +1,101 @@
+import copy
+import json
+import global_vars
+from chats import pre, post
+from pingpong import PingPong
+from gens.batch_gen import get_output_batch
+
+from pingpong.context import CtxLastWindowStrategy
+
+def build_prompts(ppmanager, user_message, global_context, win_size=3):
+    dummy_ppm = copy.deepcopy(ppmanager)
+    
+    dummy_ppm.ctx = global_context
+    for pingpong in dummy_ppm.pingpongs:
+        pong = pingpong.pong
+        first_sentence = pong.split("\n")[0]
+        if first_sentence != "" and \
+            pre.contains_image_markdown(first_sentence):
+            pong = ' '.join(pong.split("\n")[1:]).strip()
+            pingpong.pong = pong
+            
+    lws = CtxLastWindowStrategy(win_size)
+    
+    prompt = lws(dummy_ppm)
+    return prompt
+
+def text_stream(ppmanager, streamer):
+    for new_text in streamer:
+        ppmanager.append_pong(new_text)
+        yield ppmanager, ppmanager.build_uis()
+                
+    yield ppmanager, ppmanager.build_uis()
+
+def summarize(
+    ppmanager, prompt_to_summarize, win_size,
+    temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+    num_beams, use_cache, do_sample, eos_token_id, pad_token_id    
+):
+    ctx = ppmanager.ctx
+    last_pong = ppmanager.pingpongs[-1].pong
+    ppmanager.add_pingpong(PingPong(prompt_to_summarize, ""))
+    prompt = ppmanager.build_prompts(from_idx=-win_size)
+    
+    _, gen_config_summarization = pre.build_gen_config(
+        temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+        num_beams, use_cache, do_sample, eos_token_id, pad_token_id
+    )
+    summarize_output = get_output_batch(
+        global_vars.model, global_vars.tokenizer, [prompt], gen_config_summarization
+    )[0].strip()
+    ppmanager.ctx = summarize_output
+    ppmanager.pop_pingpong()
+    return ppmanager
+
+def chat_stream(
+    idx, local_data, user_message, state, model_num,
+    global_context, ctx_num_lconv, ctx_sum_prompt,
+    res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+):
+    res = [
+      state["ppmanager_type"].from_json(json.dumps(ppm))
+      for ppm in local_data
+    ]
+
+    ppm = res[idx]
+
+    # add_ping returns a prompt structured in Alpaca form
+    ppm.add_pingpong(
+        PingPong(user_message, "")
+    )
+    prompt = build_prompts(ppm, user_message, global_context, ctx_num_lconv)
+
+    # prepare text generating streamer & start generating
+    gen_kwargs, streamer = pre.build(
+        prompt,
+        res_temp, res_topp, res_topk, res_rpen, res_mnts, 
+        res_beams, res_cache, res_sample, res_eosid, res_padid,
+        return_token_type_ids=False
+    )
+    pre.start_gen(gen_kwargs)
+
+    # handling stream
+    for ppmanager, uis in text_stream(ppm, streamer):
+        yield "", uis, prompt, str(res)
+
+    ppm = post.strip_pong(ppm)
+    yield "", ppm.build_uis(), prompt, str(res)
+    
+    # summarization
+    # ppm.add_pingpong(
+    #     PingPong(None, "![](https://i.postimg.cc/ZKNKDPBd/Vanilla-1s-209px.gif)")
+    # )
+    # yield "", ppm.build_uis(), prompt, state
+    # ppm.pop_pingpong()
+    
+    # ppm = summarize(
+    #     ppm, ctx_sum_prompt, ctx_num_lconv,
+    #     sum_temp, sum_topp, sum_topk, sum_rpen, sum_mnts, 
+    #     sum_beams, sum_cache, sum_sample, sum_eosid, sum_padid
+    # )
+    yield "", ppm.build_uis(), prompt, str(res)

chats/stablelm.py

@@ -0,0 +1,112 @@
+import torch
+from transformers import StoppingCriteria, StoppingCriteriaList
+
+import copy
+import json
+import global_vars
+from chats import pre, post
+from pingpong import PingPong
+from gens.batch_gen import get_output_batch
+
+from pingpong.context import CtxLastWindowStrategy
+
+class StopOnTokens(StoppingCriteria):
+    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
+        stop_ids = [50278, 50279, 50277, 1, 0]
+        for stop_id in stop_ids:
+            if input_ids[0][-1] == stop_id:
+                return True
+        return False
+
+def build_prompts(ppmanager, user_message, global_context, win_size=3):
+    dummy_ppm = copy.deepcopy(ppmanager)
+    
+    dummy_ppm.ctx = global_context
+    for pingpong in dummy_ppm.pingpongs:
+        pong = pingpong.pong
+        first_sentence = pong.split("\n")[0]
+        if first_sentence != "" and \
+            pre.contains_image_markdown(first_sentence):
+            pong = ' '.join(pong.split("\n")[1:]).strip()
+            pingpong.pong = pong
+            
+    lws = CtxLastWindowStrategy(win_size)
+    
+    prompt = lws(dummy_ppm)
+    return prompt
+
+def text_stream(ppmanager, streamer):
+    for new_text in streamer:
+        ppmanager.append_pong(new_text)
+        yield ppmanager, ppmanager.build_uis()
+                
+    yield ppmanager, ppmanager.build_uis()
+
+def summarize(
+    ppmanager, prompt_to_summarize, win_size,
+    temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+    num_beams, use_cache, do_sample, eos_token_id, pad_token_id    
+):
+    ctx = ppmanager.ctx
+    last_pong = ppmanager.pingpongs[-1].pong
+    ppmanager.add_pingpong(PingPong(prompt_to_summarize, ""))
+    prompt = ppmanager.build_prompts(from_idx=-win_size)
+
+    _, gen_config_summarization = pre.build_gen_config(
+        temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+        num_beams, use_cache, do_sample, eos_token_id, pad_token_id
+    )
+    summarize_output = get_output_batch(
+        global_vars.model, global_vars.tokenizer, [prompt], gen_config_summarization
+    )[0].split(prompt_to_summarize)[-1].strip()
+    ppmanager.ctx = summarize_output
+    ppmanager.pop_pingpong()
+    return ppmanager
+
+def chat_stream(
+    idx, local_data, user_message, state, model_num,
+    global_context, ctx_num_lconv, ctx_sum_prompt,
+    res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+):
+    res = [
+      state["ppmanager_type"].from_json(json.dumps(ppm))
+      for ppm in local_data
+    ]
+
+    ppm = res[idx]
+
+    # add_ping returns a prompt structured in Alpaca form
+    ppm.add_pingpong(
+        PingPong(user_message, "")
+    )
+    prompt = build_prompts(ppm, user_message, global_context, ctx_num_lconv)
+
+    # prepare text generating streamer & start generating
+    gen_kwargs, streamer = pre.build(
+        prompt,
+        res_temp, res_topp, res_topk, res_rpen, res_mnts, 
+        res_beams, res_cache, res_sample, res_eosid, res_padid,
+        StoppingCriteriaList([StopOnTokens()]), False
+    )
+    pre.start_gen(gen_kwargs)
+
+    # handling stream
+    for ppmanager, uis in text_stream(ppm, streamer):
+        yield "", uis, prompt, str(res)
+
+    ppm = post.strip_pong(ppm)
+    yield "", ppm.build_uis(), prompt, str(res)
+    
+    # summarization
+    # ppm.add_pingpong(
+    #     PingPong(None, "![](https://i.postimg.cc/ZKNKDPBd/Vanilla-1s-209px.gif)")
+    # )
+    # yield "", ppm.build_uis(), prompt, state
+    # ppm.pop_pingpong()
+    
+    # ppm = summarize(
+    #     ppm, ctx_sum_prompt, ctx_num_lconv,
+    #     sum_temp, sum_topp, sum_topk, sum_rpen, sum_mnts, 
+    #     sum_beams, sum_cache, sum_sample, sum_eosid, sum_padid
+    # )
+    yield "", ppm.build_uis(), prompt, str(res)    

chats/starchat.py

@@ -0,0 +1,112 @@
+import torch
+from transformers import StoppingCriteria, StoppingCriteriaList
+
+import copy
+import json
+import global_vars
+from chats import pre, post
+from pingpong import PingPong
+from gens.batch_gen import get_output_batch
+
+from pingpong.context import CtxLastWindowStrategy
+
+class StopOnTokens(StoppingCriteria):
+    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
+        stop_ids = [49155, 1, 0]
+        for stop_id in stop_ids:
+            if input_ids[0][-1] == stop_id:
+                return True
+        return False
+
+def build_prompts(ppmanager, user_message, global_context, win_size=3):
+    dummy_ppm = copy.deepcopy(ppmanager)
+    
+    dummy_ppm.ctx = global_context
+    for pingpong in dummy_ppm.pingpongs:
+        pong = pingpong.pong
+        first_sentence = pong.split("\n")[0]
+        if first_sentence != "" and \
+            pre.contains_image_markdown(first_sentence):
+            pong = ' '.join(pong.split("\n")[1:]).strip()
+            pingpong.pong = pong
+            
+    lws = CtxLastWindowStrategy(win_size)
+    
+    prompt = lws(dummy_ppm)
+    return prompt
+
+def text_stream(ppmanager, streamer):
+    for new_text in streamer:
+        ppmanager.append_pong(new_text)
+        yield ppmanager, ppmanager.build_uis()
+                
+    yield ppmanager, ppmanager.build_uis()
+
+def summarize(
+    ppmanager, prompt_to_summarize, win_size,
+    temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+    num_beams, use_cache, do_sample, eos_token_id, pad_token_id    
+):
+    ctx = ppmanager.ctx
+    last_pong = ppmanager.pingpongs[-1].pong
+    ppmanager.add_pingpong(PingPong(prompt_to_summarize, ""))
+    prompt = ppmanager.build_prompts(from_idx=-win_size)
+    
+    _, gen_config_summarization = pre.build_gen_config(
+        temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+        num_beams, use_cache, do_sample, eos_token_id, pad_token_id
+    )
+    summarize_output = get_output_batch(
+        global_vars.model, global_vars.tokenizer, [prompt], gen_config_summarization
+    )[0].strip()
+    ppmanager.ctx = summarize_output
+    ppmanager.pop_pingpong()
+    return ppmanager
+
+def chat_stream(
+    idx, local_data, user_message, state, model_num,
+    global_context, ctx_num_lconv, ctx_sum_prompt,
+    res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+):
+    res = [
+      state["ppmanager_type"].from_json(json.dumps(ppm))
+      for ppm in local_data
+    ]
+
+    ppm = res[idx]
+
+    # add_ping returns a prompt structured in Alpaca form
+    ppm.add_pingpong(
+        PingPong(user_message, "")
+    )
+    prompt = build_prompts(ppm, user_message, global_context, ctx_num_lconv)
+
+    # prepare text generating streamer & start generating
+    gen_kwargs, streamer = pre.build(
+        prompt,
+        res_temp, res_topp, res_topk, res_rpen, res_mnts, 
+        res_beams, res_cache, res_sample, res_eosid, res_padid,
+        StoppingCriteriaList([StopOnTokens()]), False
+    )
+    pre.start_gen(gen_kwargs)
+
+    # handling stream
+    for ppmanager, uis in text_stream(ppm, streamer):
+        yield "", uis, prompt, str(res)
+
+    ppm = post.strip_pong(ppm)
+    yield "", ppm.build_uis(), prompt, str(res)
+    
+    # summarization
+    # ppm.add_pingpong(
+    #     PingPong(None, "![](https://i.postimg.cc/ZKNKDPBd/Vanilla-1s-209px.gif)")
+    # )
+    # yield "", ppm.build_uis(), prompt, state
+    # ppm.pop_pingpong()
+    
+    # ppm = summarize(
+    #     ppm, ctx_sum_prompt, ctx_num_lconv,
+    #     sum_temp, sum_topp, sum_topk, sum_rpen, sum_mnts, 
+    #     sum_beams, sum_cache, sum_sample, sum_eosid, sum_padid
+    # )
+    yield "", ppm.build_uis(), prompt, str(res)

chats/vicuna.py

@@ -0,0 +1,109 @@
+import copy
+import json
+import global_vars
+from chats import pre, post
+from pingpong import PingPong
+from gens.batch_gen import get_output_batch
+
+from pingpong.context import CtxLastWindowStrategy
+
+def build_prompts(ppmanager, user_message, global_context, win_size=3):
+    dummy_ppm = copy.deepcopy(ppmanager)
+    
+    dummy_ppm.ctx = global_context
+    for pingpong in dummy_ppm.pingpongs:
+        pong = pingpong.pong
+        first_sentence = pong.split("\n")[0]
+        if first_sentence != "" and \
+            pre.contains_image_markdown(first_sentence):
+            pong = ' '.join(pong.split("\n")[1:]).strip()
+            pingpong.pong = pong
+            
+    lws = CtxLastWindowStrategy(win_size)
+    
+    prompt = lws(dummy_ppm)
+    return prompt
+
+def text_stream(ppmanager, streamer, model_thumbnail_tiny, model_type):
+    count = 0
+    
+    for new_text in streamer:
+        if count == 0:
+            ppmanager.append_pong(f"![]({model_thumbnail_tiny})***[{model_type}]***\n")
+            count = count + 1        
+        
+        ppmanager.append_pong(new_text)
+        yield ppmanager, ppmanager.build_uis()
+                
+    yield ppmanager, ppmanager.build_uis()
+
+def summarize(
+    ppmanager, prompt_to_summarize, win_size,
+    temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+    num_beams, use_cache, do_sample, eos_token_id, pad_token_id    
+):
+    ctx = ppmanager.ctx
+    last_pong = ppmanager.pingpongs[-1].pong
+    ppmanager.add_pingpong(PingPong(prompt_to_summarize, ""))
+    prompt = ppmanager.build_prompts(from_idx=-win_size)
+    
+    _, gen_config_summarization = pre.build_gen_config(
+        temperature, top_p, top_k, repetition_penalty, max_new_tokens,
+        num_beams, use_cache, do_sample, eos_token_id, pad_token_id
+    )
+    summarize_output = get_output_batch(
+        global_vars.model, global_vars.tokenizer, [prompt], gen_config_summarization
+    )[0].strip()
+    ppmanager.ctx = summarize_output
+    ppmanager.pop_pingpong()
+    return ppmanager
+
+def chat_stream(
+    idx, local_data, user_message, state, model_num,
+    global_context, ctx_num_lconv, ctx_sum_prompt,
+    res_temp, res_topp, res_topk, res_rpen, res_mnts, res_beams, res_cache, res_sample, res_eosid, res_padid,
+):
+    res = [
+      state["ppmanager_type"].from_json(json.dumps(ppm))
+      for ppm in local_data
+    ]
+
+    ppm = res[idx]
+
+    # add_ping returns a prompt structured in Alpaca form
+    ppm.add_pingpong(
+        PingPong(user_message, "")
+    )
+    prompt = build_prompts(ppm, user_message, global_context, ctx_num_lconv)
+
+    # prepare text generating streamer & start generating
+    gen_kwargs, streamer = pre.build(
+        prompt, model_num,
+        res_temp, res_topp, res_topk, res_rpen, res_mnts, 
+        res_beams, res_cache, res_sample, res_eosid, res_padid,
+        return_token_type_ids=False
+    )
+    pre.start_gen(gen_kwargs, model_num)
+
+    # handling stream
+    model_thumbnail_tiny = global_vars.models[model_num]["model_thumb_tiny"]
+    model_type = global_vars.models[model_num]["model_type"]
+    for ppmanager, uis in text_stream(ppm, streamer, model_thumbnail_tiny, model_type):
+        yield "", uis, prompt, str(res)
+
+    ppm = post.strip_pong(ppm)
+    yield "", ppm.build_uis(), prompt, str(res)
+    
+    # summarization
+    # ppm.add_pingpong(
+    #     PingPong(None, "![](https://i.postimg.cc/ZKNKDPBd/Vanilla-1s-209px.gif)")
+    # )
+    # yield "", ppm.build_uis(), prompt, state
+    # ppm.pop_pingpong()
+    
+    # ppm = summarize(
+    #     ppm, ctx_sum_prompt, ctx_num_lconv,
+    #     sum_temp, sum_topp, sum_topk, sum_rpen, sum_mnts, 
+    #     sum_beams, sum_cache, sum_sample, sum_eosid, sum_padid
+    # )
+    yield "", ppm.build_uis(), prompt, str(res)

configs/constraints_config.yaml

@@ -0,0 +1,4 @@
+constraints:
+    max_context: 1000
+    max_prompt: 300
+    max_conv_len: 1500

configs/response_configs/baize.yaml

@@ -0,0 +1,12 @@
+generation_config:
+    temperature: 0.95
+    top_p: 0.9
+    top_k: 50
+    num_beams: 1
+    use_cache: True
+    repetition_penalty: 1.2
+    max_new_tokens: 1024
+    do_sample: True
+    bos_token_id: 0
+    eos_token_id: 1
+    pad_token_id: 0

configs/response_configs/camel.yaml

@@ -0,0 +1,11 @@
+generation_config:
+    temperature: 0.95
+    top_p: 0.9
+    top_k: 50
+    num_beams: 1
+    use_cache: True
+    repetition_penalty: 1.2
+    max_new_tokens: 1024
+    do_sample: True
+    pad_token_id: 50257
+    eos_token_id: 50256

configs/response_configs/default.yaml

@@ -0,0 +1,9 @@
+generation_config:
+    temperature: 0.95
+    top_p: 0.9
+    top_k: 50
+    num_beams: 1
+    use_cache: True
+    repetition_penalty: 1.2
+    max_new_tokens: 1024
+    do_sample: True

configs/response_configs/flan.yaml

@@ -0,0 +1,9 @@
+generation_config:
+    temperature: 0.8
+    top_p: 0.95
+    top_k: 50
+    num_beams: 1
+    use_cache: False
+    repetition_penalty: 1.2
+    max_new_tokens: 256
+    do_sample: True

configs/response_configs/gpt4_alpaca.yaml

@@ -0,0 +1,9 @@
+generation_config:
+    temperature: 0.95
+    top_p: 0.9
+    top_k: 50
+    num_beams: 1
+    use_cache: True
+    repetition_penalty: 1.2
+    max_new_tokens: 512
+    do_sample: True

configs/response_configs/guanaco.yaml

@@ -0,0 +1,9 @@
+generation_config:
+    temperature: 0.95
+    top_p: 0.9
+    top_k: 50
+    num_beams: 1
+    use_cache: True
+    repetition_penalty: 1.2
+    max_new_tokens: 1024
+    do_sample: True

configs/response_configs/koalpaca.yaml

@@ -0,0 +1,11 @@
+generation_config:
+    temperature: 0.95
+    top_p: 0.9
+    top_k: 50
+    num_beams: 1
+    use_cache: True
+    repetition_penalty: 1.2
+    max_new_tokens: 1024
+    do_sample: True
+    eos_token_id: 2
+    pad_token_id: 2

configs/response_configs/redpajama.yaml

@@ -0,0 +1,11 @@
+generation_config:
+    temperature: 1.0
+    top_p: 0.9
+    top_k: 1000
+    num_beams: 1
+    use_cache: True
+    repetition_penalty: 1.2
+    max_new_tokens: 512
+    do_sample: True
+    eos_token_id: 0
+    pad_token_id: 1

configs/response_configs/stablelm.yaml

@@ -0,0 +1,11 @@
+generation_config:
+    temperature: 1.0
+    top_p: 0.9
+    top_k: 1000
+    num_beams: 1
+    use_cache: True
+    repetition_penalty: 1.2
+    max_new_tokens: 512
+    do_sample: True
+    eos_token_id: 0
+    pad_token_id: 1

configs/response_configs/stackllama.yaml

@@ -0,0 +1,10 @@
+generation_config:
+    temperature: 0.9
+    top_p: 0.95
+    # top_k: 50
+    num_beams: 1
+    use_cache: True
+    repetition_penalty: 1.2
+    max_new_tokens: 256
+    do_sample: True
+    early_stopping: True

configs/response_configs/starchat.yaml

@@ -0,0 +1,12 @@
+generation_config:
+    temperature: 0.5
+    top_p: 0.95
+    top_k: 50
+    num_beams: 1
+    use_cache: True
+    repetition_penalty: 1.2
+    max_new_tokens: 1024
+    do_sample: True
+    eos_token_id: 0
+    bos_token_id: 0
+    pad_token_id: 0

configs/response_configs/t5_vicuna.yaml

@@ -0,0 +1,9 @@
+generation_config:
+    temperature: 0.95
+    top_p: 0.9
+    top_k: 50
+    num_beams: 1
+    use_cache: True
+    repetition_penalty: 1.2
+    max_new_tokens: 2048
+    do_sample: True

configs/summarization_configs/camel.yaml

@@ -0,0 +1,11 @@
+generation_config:
+    temperature: 1
+    top_p: 0.9
+    top_k: 50
+    num_beams: 1
+    use_cache: True
+    max_new_tokens: 1024
+    do_sample: True
+    repetition_penalty: 1.5    
+    pad_token_id: 50257
+    eos_token_id: 50256

configs/summarization_configs/default.yaml

@@ -0,0 +1,11 @@
+generation_config:
+    temperature: 1
+    top_p: 0.9
+    top_k: 50
+    num_beams: 1
+    use_cache: True
+    max_new_tokens: 1024
+    do_sample: True
+    repetition_penalty: 1.5
+    
+    

configs/summarization_configs/koalpaca.yaml

@@ -0,0 +1,11 @@
+generation_config:
+    temperature: 1
+    top_p: 0.9
+    top_k: 50
+    num_beams: 1
+    use_cache: True
+    do_sample: True
+    repetition_penalty: 1.2
+    max_new_tokens: 512
+    eos_token_id: 2
+    pad_token_id: 2

configs/summarization_configs/redpajama.yaml

@@ -0,0 +1,11 @@
+generation_config:
+    temperature: 1.0
+    top_p: 0.9
+    top_k: 1000
+    num_beams: 1
+    use_cache: True
+    repetition_penalty: 1.2
+    max_new_tokens: 512
+    do_sample: True
+    eos_token_id: 0
+    pad_token_id: 1

configs/summarization_configs/stablelm.yaml

@@ -0,0 +1,11 @@
+generation_config:
+    temperature: 1
+    top_p: 0.9
+    top_k: 1000
+    num_beams: 1
+    use_cache: True
+    do_sample: True
+    repetition_penalty: 1.2
+    max_new_tokens: 512
+    eos_token_id: 0
+    pad_token_id: 1

configs/summarization_configs/t5_vicuna.yaml

@@ -0,0 +1,9 @@
+generation_config:
+    temperature: 0.95
+    top_p: 0.9
+    top_k: 50
+    num_beams: 1
+    use_cache: True
+    repetition_penalty: 1.2
+    max_new_tokens: 2048
+    do_sample: True

examples.txt

@@ -0,0 +1,2 @@
+Tell me about GPT
+Write a Python program to print Fibonacci numbers

gens/batch_gen.py

@@ -0,0 +1,32 @@
+import torch
+
+def get_output_batch(
+    model, tokenizer, prompts, generation_config, device='cuda'
+):
+    if len(prompts) == 1:
+        encoding = tokenizer(prompts, return_tensors="pt")
+        input_ids = encoding["input_ids"].to(device)
+        generated_id = model.generate(
+            input_ids=input_ids,
+            generation_config=generation_config,
+        )
+
+        decoded = tokenizer.batch_decode(
+            generated_id, skip_prompt=True, skip_special_tokens=True
+        )
+        del input_ids, generated_id
+        torch.cuda.empty_cache()
+        return decoded
+    else:
+        encodings = tokenizer(prompts, padding=True, return_tensors="pt").to(device)
+        generated_ids = model.generate(
+            **encodings,
+            generation_config=generation_config,
+        )
+
+        decoded = tokenizer.batch_decode(
+            generated_ids, skip_prompt=True, skip_special_tokens=True
+        )
+        del encodings, generated_ids
+        torch.cuda.empty_cache()
+        return decoded

global_vars.py

@@ -0,0 +1,194 @@
+import gc
+import yaml
+import json
+import torch
+from transformers import GenerationConfig
+from models import alpaca, stablelm, koalpaca, flan_alpaca, mpt
+from models import camel, t5_vicuna, vicuna, starchat, redpajama, bloom
+from models import baize, guanaco, falcon, kullm, replit, airoboros
+from models import samantha_vicuna
+
+from utils import get_chat_interface, get_chat_manager
+
+model_infos = json.load(open("model_cards.json"))
+
+def get_model_type(model_info):
+    base_url = model_info["hub(base)"]
+    ft_ckpt_url = model_info["hub(ckpt)"]
+    
+    model_type_tmp = "alpaca"
+    if "llms/wizardlm" in base_url.lower():
+        model_type_tmp = "wizardlm"
+    elif "chronos" in base_url.lower():
+        model_type_tmp = "chronos"
+    elif "lazarus" in base_url.lower():
+        model_type_tmp = "lazarus"
+    elif "samantha" in base_url.lower():
+        model_type_tmp = "samantha-vicuna"
+    elif "airoboros" in base_url.lower():
+        model_type_tmp = "airoboros"
+    elif "replit" in base_url.lower():
+        model_type_tmp = "replit-instruct"
+    elif "kullm" in base_url.lower():
+        model_type_tmp = "kullm-polyglot"
+    elif "nous-hermes" in base_url.lower():
+        model_type_tmp = "nous-hermes"
+    elif "guanaco" in base_url.lower():
+        model_type_tmp = "guanaco"
+    elif "wizardlm-uncensored-falcon" in base_url.lower():
+        model_type_tmp = "wizard-falcon"        
+    elif "falcon" in base_url.lower():
+        model_type_tmp = "falcon"
+    elif "baize" in base_url.lower():
+        model_type_tmp = "baize"
+    elif "stable-vicuna" in base_url.lower():
+        model_type_tmp = "stable-vicuna"        
+    elif "vicuna" in base_url.lower():
+        model_type_tmp = "vicuna"
+    elif "mpt" in base_url.lower():
+        model_type_tmp = "mpt"
+    elif "redpajama-incite-7b-instruct" in base_url.lower():
+        model_type_tmp = "redpajama-instruct"
+    elif "redpajama" in base_url.lower():
+        model_type_tmp = "redpajama"
+    elif "starchat" in base_url.lower():
+        model_type_tmp = "starchat"
+    elif "camel" in base_url.lower():
+        model_type_tmp = "camel"
+    elif "flan-alpaca" in base_url.lower():
+        model_type_tmp = "flan-alpaca"
+    elif "openassistant/stablelm" in base_url.lower():
+        model_type_tmp = "os-stablelm"
+    elif "stablelm" in base_url.lower():
+        model_type_tmp = "stablelm"
+    elif "fastchat-t5" in base_url.lower():
+        model_type_tmp = "t5-vicuna"
+    elif "koalpaca-polyglot" in base_url.lower():
+        model_type_tmp = "koalpaca-polyglot"
+    elif "alpacagpt4" in ft_ckpt_url.lower():
+        model_type_tmp = "alpaca-gpt4"
+    elif "alpaca" in ft_ckpt_url.lower():
+        model_type_tmp = "alpaca"
+    elif "llama-deus" in ft_ckpt_url.lower():
+        model_type_tmp = "llama-deus"
+    elif "vicuna-lora-evolinstruct" in ft_ckpt_url.lower():
+        model_type_tmp = "evolinstruct-vicuna"
+    elif "alpacoom" in ft_ckpt_url.lower():
+        model_type_tmp = "alpacoom"
+    elif "guanaco" in ft_ckpt_url.lower():
+        model_type_tmp = "guanaco"
+    else:
+        print("unsupported model type")
+        
+    return model_type_tmp
+
+def initialize_globals():
+    global models, tokenizers
+    
+    models = []
+    model_names = [
+        "baize-7b",   
+        # "evolinstruct-vicuna-13b",
+        "guanaco-7b",
+        # "nous-hermes-13b"
+    ]
+    for model_name in model_names:
+        model_info = model_infos[model_name]
+        model_thumbnail_tiny = model_info["thumb-tiny"]
+        model_type = get_model_type(model_info)
+        print(model_type)
+        load_model = get_load_model(model_type)
+        
+        model, tokenizer = load_model(
+            base=model_info["hub(base)"],
+            finetuned=model_info["hub(ckpt)"],
+            mode_cpu=False,
+            mode_mps=False,
+            mode_full_gpu=True,
+            mode_8bit=False,
+            mode_4bit=False,
+            force_download_ckpt=False
+        )        
+        
+        gen_config, gen_config_raw = get_generation_config(
+            model_info["default_gen_config"]
+        )
+        
+        models.append(
+            {
+                "model_name": model_name,
+                "model_thumb_tiny": model_thumbnail_tiny,
+                "model_type": model_type,
+                "model": model,
+                "tokenizer": tokenizer,
+                "gen_config": gen_config,
+                "gen_config_raw": gen_config_raw,
+                "chat_interface": get_chat_interface(model_type),
+                "chat_manager": get_chat_manager(model_type),
+            }
+        )
+        
+def get_load_model(model_type):
+    if model_type == "alpaca" or \
+        model_type == "alpaca-gpt4" or \
+        model_type == "llama-deus" or \
+        model_type == "nous-hermes" or \
+        model_type == "lazarus" or \
+        model_type == "chronos" or \
+        model_type == "wizardlm":
+        return alpaca.load_model
+    elif model_type == "stablelm" or model_type == "os-stablelm":
+        return stablelm.load_model
+    elif model_type == "koalpaca-polyglot":
+        return koalpaca.load_model
+    elif model_type == "kullm-polyglot":
+        return kullm.load_model
+    elif model_type == "flan-alpaca":
+        return flan_alpaca.load_model
+    elif model_type == "camel":
+        return camel.load_model
+    elif model_type == "t5-vicuna":
+        return t5_vicuna.load_model
+    elif model_type == "stable-vicuna":
+        return vicuna.load_model
+    elif model_type == "starchat":
+        return starchat.load_model
+    elif model_type == "mpt":
+        return mpt.load_model
+    elif model_type == "redpajama" or \
+        model_type == "redpajama-instruct":
+        return redpajama.load_model
+    elif model_type == "vicuna":
+        return vicuna.load_model
+    elif model_type == "evolinstruct-vicuna":
+        return alpaca.load_model
+    elif model_type == "alpacoom":
+        return bloom.load_model
+    elif model_type == "baize":
+        return baize.load_model
+    elif model_type == "guanaco":
+        return guanaco.load_model
+    elif model_type == "falcon" or model_type == "wizard-falcon":
+        return falcon.load_model
+    elif model_type == "replit-instruct":
+        return replit.load_model
+    elif model_type == "airoboros":
+        return airoboros.load_model
+    elif model_type == "samantha-vicuna":
+        return samantha_vicuna.load_model
+    else:
+        return None
+    
+def get_generation_config(path):
+    with open(path, 'rb') as f:
+        generation_config = yaml.safe_load(f.read())
+        
+    generation_config = generation_config["generation_config"]
+
+    return GenerationConfig(**generation_config), generation_config
+
+def get_constraints_config(path):
+    with open(path, 'rb') as f:
+        constraints_config = yaml.safe_load(f.read())
+        
+    return ConstraintsConfig(**constraints_config), constraints_config["constraints"]

miscs/js.py

@@ -0,0 +1,50 @@
+GET_LOCAL_STORAGE = """
+function() {
+  globalThis.setStorage = (key, value)=>{
+    localStorage.setItem(key, JSON.stringify(value));
+  }
+  globalThis.getStorage = (key, value)=>{
+    return JSON.parse(localStorage.getItem(key));
+  }
+
+  var local_data = getStorage('local_data');
+  var history = [];
+
+  if(local_data) {
+    local_data[0].pingpongs.forEach(element =>{ 
+      history.push([element.ping, element.pong]);
+    });
+  }
+  else {
+    local_data = [];
+    for (let step = 0; step < 10; step++) {
+      local_data.push({'ctx': '', 'pingpongs':[]});
+    }
+    setStorage('local_data', local_data);
+  }
+
+  if(history.length == 0) {
+    document.querySelector("#initial-popup").classList.remove('hide');
+  }
+  
+  return [history, local_data];
+}
+"""
+
+UPDATE_LEFT_BTNS_STATE = """
+(v)=>{
+  document.querySelector('.custom-btn-highlight').classList.add('custom-btn');
+  document.querySelector('.custom-btn-highlight').classList.remove('custom-btn-highlight');
+
+  const elements = document.querySelectorAll(".custom-btn");
+
+  for(var i=0; i < elements.length; i++) {
+    const element = elements[i];
+    if(element.textContent == v) {
+      console.log(v);
+      element.classList.add('custom-btn-highlight');
+      element.classList.remove('custom-btn');
+      break;
+    }
+  }
+}""" 

miscs/strings.py

@@ -0,0 +1,83 @@
+TITLE = "Alpaca-LoRA Playground"
+
+ABSTRACT = """
+Thanks to [tolen](https://github.com/tloen/alpaca-lora), this application runs Alpaca-LoRA which is instruction fine-tuned version of [LLaMA](https://ai.facebook.com/blog/large-language-model-llama-meta-ai/). This demo currently runs 30B version on a 3*A6000 instance at [Jarvislabs.ai](https://jarvislabs.ai/).
+
+NOTE: too long input (context, instruction) will not be allowed. Please keep context < 500 and instruction < 150
+"""
+
+BOTTOM_LINE = """
+This demo application runs the open source project, [Alpaca-LoRA-Serve](https://github.com/deep-diver/Alpaca-LoRA-Serve). By default, it runs with streaming mode, but you can also run with dynamic batch generation model. Please visit the repo, find more information, and contribute if you can.
+
+Alpaca-LoRA is built on the same concept as Standford Alpaca project, but it lets us train and inference on a smaller GPUs such as RTX4090 for 7B version. Also, we could build very small size of checkpoints on top of base models thanks to [🤗 transformers](https://huggingface.co/docs/transformers/index), [🤗 peft](https://github.com/huggingface/peft), and [bitsandbytes](https://github.com/TimDettmers/bitsandbytes/tree/main) libraries.
+
+We are thankful to the [Jarvislabs.ai](https://jarvislabs.ai/) who generously provided free GPU instances.
+"""
+
+DEFAULT_EXAMPLES = {
+    "Typical Questions": [
+        {
+            "title": "List all Canadian provinces in alphabetical order.",
+            "examples": [
+                ["1", "List all Canadian provinces in alphabetical order."],
+                ["2", "Which ones are on the east side?"],
+                ["3", "What foods are famous in each province on the east side?"],
+                ["4", "What about sightseeing? or landmarks? list one per province"],
+            ],
+        },
+        {
+            "title": "Tell me about Alpacas.",
+            "examples": [
+                ["1", "Tell me about alpacas in two sentences"],
+                ["2", "What other animals are living in the same area?"],
+                ["3", "Are they the same species?"],
+                ["4", "Write a Python program to return those species"],
+            ],
+        },
+        {
+            "title": "Tell me about the king of France in 2019.",
+            "examples": [
+                ["1", "Tell me about the king of France in 2019."],
+                ["2", "What about before him?"],
+            ]
+        },
+        {
+            "title": "Write a Python program that prints the first 10 Fibonacci numbers.",
+            "examples": [
+                ["1", "Write a Python program that prints the first 10 Fibonacci numbers."],
+                ["2", "Could you explain how the code works?"],
+                ["3", "What is recursion?"],
+            ]
+        }
+    ],
+    "Identity": [
+        {
+            "title": "Conversation with the planet Pluto",
+            "examples": [
+                ["1", "Conversation with the planet Pluto", "I'am so curious about you"],
+                ["2", "Conversation with the planet Pluto", "Tell me what I would see if I visited"],
+                ["3", "Conversation with the planet Pluto", "It sounds beautiful"],
+                ["4", "Conversation with the planet Pluto", "I'll keep that in mind. Hey I was wondering have you ever had any visitor?"],
+                ["5", "Conversation with the planet Pluto", "That must have been exciting"],
+                ["6", "Conversation with the planet Pluto", "That's so great. What else do you wish people knew about you?"],
+                ["7", "Conversation with the planet Pluto", "Thanks for talking with me"],
+            ]
+        },
+        {
+            "title": "Conversation with a paper airplane",
+            "examples": [
+                ["1", "Conversation with a paper airplane", "What's it like being thrown through the air"],
+                ["2", "Conversation with a paper airplane", "What's the worst place you've ever landed"],
+                ["3", "Conversation with a paper airplane", "Have you ever stucked?"],
+                ["4", "Conversation with a paper airplane", "What's the secret to a really good paper airplane?"],
+                ["5", "Conversation with a paper airplane", "What's the farthest you've ever flown?"],
+                ["6", "Conversation with a paper airplane", "Good to talk to you!"]
+            ]
+        }
+    ]
+}
+
+SPECIAL_STRS = {
+    "continue": "continue.",
+    "summarize": "what have we discussed so far? describe in the user's view and include important entities. also be brief as much as possible."
+}

miscs/styles.py

@@ -0,0 +1,727 @@
+PARENT_BLOCK_CSS = """
+#col-container {
+    width: 95%; 
+    height: 100%;
+    margin-left: auto; 
+    margin-right: auto;
+}
+
+#chatbot {
+    height: 800px; 
+    overflow: auto;
+}
+
+#chatbot > .wrap {
+    max-height: 780px;
+}
+"""
+
+MODEL_SELECTION_CSS = """
+
+.message {
+    margin: 0px !important;
+}
+
+.load-mode-selector:nth-child(3) {
+    margin: auto !important;
+    text-align: center !important;
+    width: fit-content !important;
+}
+
+code {
+    white-space: break-spaces !important;
+}
+
+.progress-view {
+    background: transparent !important;
+    border-radius: 25px !important;
+}
+
+#landing-container {
+    width: 85%;
+    margin: auto;
+}
+
+.landing-btn {
+    font-size: 2.3vw !important;
+    margin-top: 25px !important;
+    border-radius: 25px !important;
+    height: 120px !important;
+    
+    @media screen and (max-width: 1000px) {
+        font-size: 20px !important;
+    }
+}
+
+#landing-bottom {
+    margin-top: 20px !important;
+}
+
+.custom-btn {
+    border: none !important;
+    background: none !important;
+    box-shadow: none !important;
+    display: block !important;
+    text-align: left !important;
+}
+.custom-btn:hover {
+    background: rgb(243 244 246) !important;
+}
+
+.custom-btn-highlight {
+    border: none !important;
+    background: rgb(243 244 246) !important;
+    box-shadow: none !important;
+    display: block !important;
+    text-align: left !important;
+
+    @media (prefers-color-scheme: dark) {
+      background-color: rgba(17,24,39,255) !important;
+    }
+}
+
+#prompt-txt > label > span {
+    display: none !important;
+}
+#prompt-txt > label > textarea {
+    border: transparent;
+    border-radius: 20px;
+}
+#chatbot {
+    height: 800px; 
+    overflow: auto;
+    box-shadow: none !important;
+    border: none !important;
+}
+#chatbot > .wrap {
+    max-height: 780px;
+}
+#chatbot + div {
+  border-radius: 35px !important;
+  width: 80% !important;
+  margin: auto !important;  
+}
+
+#left-pane {
+  background-color: #f9fafb;
+  border-radius: 15px;
+  padding: 10px;
+
+  @media (prefers-color-scheme: dark) {
+    background-color: rgba(31,41,55,255) !important;
+  }  
+}
+
+#left-top {
+  padding-left: 10px;
+  padding-right: 10px;
+  text-align: center;
+  font-weight: bold;
+  font-size: large;    
+}
+
+#chat-history-accordion {
+  background: transparent;
+  border: 0.8px !important;  
+}
+
+#right-pane {
+  margin-left: 20px;
+  background: white;
+  border-radius: 20px;
+
+  @media (prefers-color-scheme: dark) {
+    background-color: rgba(31,41,55,255) !important;
+  }
+  
+  @media screen and (max-width: 1000px) {
+      margin: 0px !important;
+  }
+}
+
+#initial-popup {
+  z-index: 100;
+  position: absolute;
+  width: 50%;
+  top: 50%;
+  height: 50%;
+  left: 50%;
+  transform: translate(-50%, -50%);
+  border-radius: 35px;
+  padding: 15px;
+}
+
+#initial-popup-title {
+  text-align: center;
+  font-size: 18px;
+  font-weight: bold;    
+}
+
+#initial-popup-left-pane {
+  min-width: 150px !important;
+}
+
+#initial-popup-right-pane {
+  text-align: right;
+}
+
+.example-btn {
+  padding-top: 20px !important;
+  padding-bottom: 20px !important;
+  padding-left: 5px !important;
+  padding-right: 5px !important;
+  background: linear-gradient(to bottom right, #f7faff, #ffffff) !important;
+  box-shadow: none !important;
+  border-radius: 20px !important;
+
+  @media (prefers-color-scheme: dark) {
+    background: rgba(70,79,86,255) !important;
+  }
+}
+
+.example-btn:hover {
+  box-shadow: 0.3px 0.3px 0.3px gray !important;
+
+  @media (prefers-color-scheme: dark) {
+    background: rgba(34,37,42,255) !important;
+  }
+}
+
+.example-btn:active {
+  @media (prefers-color-scheme: dark) {
+    background: rgba(70,79,86,255) !important;
+  }
+}
+
+#example-title {
+  margin-bottom: 15px;
+}
+
+#aux-btns-popup {
+    z-index: 200;
+    position: absolute !important;
+    bottom: 75px !important;
+    right: 40px !important;
+}
+
+#aux-btns-popup > div {
+    flex-wrap: nowrap;
+    width: fit-content;
+    margin: auto;  
+}
+
+.aux-btn {
+    height: 30px !important;
+    flex-wrap: initial !important;
+    flex: none !important;
+    min-width: min(100px,100%) !important;
+    font-weight: unset !important;
+    font-size: 10pt !important;
+
+    background: linear-gradient(to bottom right, #f7faff, #ffffff) !important;
+    box-shadow: none !important;
+    border-radius: 20px !important;
+    
+    opacity: 0.5;
+    border-width: 0.5px;
+    border-color: grey;
+
+    @media (prefers-color-scheme: dark) {
+      opacity: 0.2 !important;
+      color: black !important;
+    }
+}
+
+.aux-btn:hover {
+    opacity: 1.0;
+    box-shadow: 0.3px 0.3px 0.3px gray !important;
+
+    @media (prefers-color-scheme: dark) {
+      opacity: 1.0 !important;
+      box-shadow: 0.3px 0.3px 0.3px gray !important;
+    }    
+}
+
+#aux-viewer {
+    position: absolute !important;
+    border-style: solid !important;
+    overflow: visible !important;
+    border: none !important;
+    box-shadow: none !important;
+    z-index: 1000 !important;
+    opacity: 0.0 !important;
+    width: 75% !important;
+    right: 1px !important; 
+    transition: all 0.5s;
+}
+
+#aux-viewer:hover {
+  opacity: 1.0 !important;
+  box-shadow: 0px 0.5px 0px 0px gray !important;
+}
+
+#aux-viewer > .label-wrap {
+  justify-content: end;
+}
+
+#aux-viewer > .label-wrap > span {
+  margin-right: 10px;
+}
+
+#aux-viewer-inspector {
+  padding: 0px;
+}
+
+#aux-viewer-inspector > label > span {
+  display: none !important;
+}
+
+#aux-viewer-inspector > label > textarea {
+  box-shadow: none;
+  border-color: transparent;
+}
+
+#global-context > label > span {
+  display: none !important;
+}
+
+#chat-back-btn {
+  background: transparent !important;
+}
+
+#chat-back-btn:hover {
+  @media (prefers-color-scheme: dark) {
+    background: rgb(75,85,99) !important;
+  }
+}
+
+#chat-back-btn:active {
+  @media (prefers-color-scheme: dark) {
+    background: transparent !important;
+  }
+}
+
+#col-container {
+    max-width: 70%; 
+    height: 100%;
+    margin-left: auto; 
+    margin-right: auto;
+}
+
+
+#container {
+  max-width: 70%;
+  margin: auto;
+
+  @media screen and (max-width: 1000px) {
+      max-width: 90% !important;
+  }
+}
+
+#container2 {
+  max-width: 60%;
+  margin: auto;
+}
+
+#container3 {
+  max-width: 60%;
+  margin: auto;
+}
+
+.square {
+  height: 100px;
+
+  @media (prefers-color-scheme: dark) {
+    background-color: rgba(70,79,86,255) !important;
+  }
+}
+
+.square:hover {
+  @media (prefers-color-scheme: dark) {
+    background-color: rgba(34,37,42,255) !important;
+  }
+}
+
+.square:active {
+  @media (prefers-color-scheme: dark) {
+    background-color: rgba(70,79,86,255) !important;
+  }
+}
+
+.placeholders {
+  min-width: max-content !important;
+}
+
+.placeholders > button {
+  border-color: transparent !important;
+  background-color: transparent !important;
+  box-shadow: none !important;
+  cursor: default !important;
+}
+
+.center {
+  text-align: center;
+  overflow: hidden;
+}
+
+#30b-placeholder1, #30b-placeholder2, #30b-placeholder3, #30b-placeholder4 {
+  background: red;
+  box-shadow: none;
+  pointer-events: none;
+  width: 100px;
+  height: 100px;
+  background: transparent !important;
+  border-color: transparent !important;
+  box-shadow: none !important;
+  cursor: default !important;
+}
+
+#20b-placeholder1, #20b-placeholder2, #20b-placeholder3, #20b-placeholder4 {
+  background: red;
+  box-shadow: none;
+  pointer-events: none;
+  width: 100px;
+  height: 100px;
+  margin: auto;
+  background: transparent !important;
+  border-color: transparent !important;
+  box-shadow: none !important;
+  cursor: default !important;
+}
+
+#10b-placeholder1, #10b-placeholder3, #10b-placeholder3, #10b-placeholder4 {
+  background: red;
+  box-shadow: none;
+  pointer-events: none;
+  width: 100px;
+  height: 100px;
+  margin: auto;
+  background: transparent !important;
+  border-color: transparent !important;
+  box-shadow: none !important;
+  cursor: default !important;
+}
+
+#camel-5b, #camel-20b {
+  background: url(https://i.ibb.co/qD5HN9T/camel-removebg-preview.png);
+  background-repeat: no-repeat;
+  background-size: 100px 100px;
+  color: transparent;
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#alpaca-lora-7b, #alpaca-lora-13b {
+  background: url(https://i.ibb.co/z89FTz2/alpaca-lora.png);
+  background-repeat: no-repeat;
+  background-size: 100px 100px;
+  color: transparent;
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#stablelm-7b {
+  background: url(https://i.ibb.co/d2pd5wk/stable-LM-cropped.png);
+  background-repeat: no-repeat;
+  background-size: 100px 100px;
+  color: transparent;
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#stackllama-7b {
+  background: url(https://i.ibb.co/Q9vLcYm/tuxpi-com-1682256296-removebg-preview.png);
+  background-repeat: no-repeat;
+  background-size: 100px 100px;
+  color: transparent;
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#flan-3b, #flan-11b {
+  background: url(https://i.ibb.co/yBTk5bv/flan.png);
+  background-repeat: no-repeat;
+  background-size: 100px 100px;
+  color: transparent;
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#koalpaca {
+  background: url(https://i.ibb.co/hF9NL7r/koalpaca.png);
+  background-repeat: no-repeat;
+  background-size: 100px 100px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#kullm {
+  background: url(https://i.ibb.co/6ZFqk4J/kullm.png);
+  background-repeat: no-repeat;
+  background-size: 100px 100px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#flan-3b {
+  background: url(https://i.ibb.co/yBTk5bv/flan.png);
+  background-repeat: no-repeat;
+  background-size: 100px 100px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#os-stablelm-7b {
+  background: url(https://i.ibb.co/WszrtVV/stablelm-oasst1.png);
+  background-repeat: no-repeat;
+  background-size: 100px 95px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#t5-vicuna-3b {
+  background: url(https://i.ibb.co/4W7n78b/chansung-vector-logo-of-collective-intelligence-of-cute-llamas-3ef46884-72e6-44da-b88a-e831e5fee747.png);
+  background-repeat: no-repeat;
+  background-size: 100px 95px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#gpt4-alpaca-7b, #gpt4-alpaca-13b {
+  background: url(https://i.ibb.co/qDz3HCG/chansung-vector-logo-of-alpaca-made-out-of-machines-Side-shot-39b27595-8202-48a6-97d1-266a745b2a29-r.png);
+  background-repeat: no-repeat;
+  background-size: 100px 95px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#stable-vicuna-13b {
+  background: url(https://i.ibb.co/b6Vv6Jh/sv.png);
+  background-repeat: no-repeat;
+  background-size: 100px 95px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;  
+}
+
+#starchat-15b, #starchat-beta-15b { 
+  background: url(https://i.ibb.co/QjPP0Vv/starcoder.png);
+  background-repeat: no-repeat;
+  background-size: 100px 95px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#redpajama-7b, #redpajama-instruct-7b {
+  background: url(https://i.ibb.co/NNB6qPj/redpajama.png);
+  background-repeat: no-repeat;
+  background-size: 100px 95px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#mpt-7b {
+  background: url(https://i.ibb.co/DwN44Z9/mpt.png);
+  background-repeat: no-repeat;
+  background-size: 100px 95px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#vicuna-7b, #vicuna-13b {
+  background: url(https://i.ibb.co/vqPDrPQ/vicuna.png);
+  background-repeat: no-repeat;
+  background-size: 100px 95px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#llama-deus-7b {
+  background: url(https://i.ibb.co/4mH9LRQ/llama-deus.png);
+  background-repeat: no-repeat;
+  background-size: 100px 95px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;  
+}
+
+#evolinstruct-vicuna-7b, #evolinstruct-vicuna-13b {
+  background: url(https://i.ibb.co/xHDRjLS/evol-vicuna.png);
+  background-repeat: no-repeat;
+  background-size: 100px 95px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#alpacoom-7b {
+  background: url(https://huggingface.co/mrm8488/Alpacoom/resolve/main/alpacoom_logo__1___1___1_-removebg-preview.png);
+  background-repeat: no-repeat;
+  background-size: 100px 95px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#baize-7b, #baize-13b {
+  background: url(https://i.ibb.co/j5VpHb0/baize.png);
+  background-repeat: no-repeat;
+  background-size: 100px 95px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#guanaco-7b, #guanaco-13b, #guanaco-33b, #guanaco-65b {
+  background: url(https://i.ibb.co/DWWsZn7/guanaco.png);
+  background-repeat: no-repeat;
+  background-size: 100px 95px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#falcon-7b, #falcon-40b {
+  background: url(https://i.ibb.co/86yNWwG/falcon.png);
+  background-repeat: no-repeat;
+  background-size: 100px 95px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#wizard-falcon-7b, #wizard-falcon-40b {
+  background: url(https://i.ibb.co/415s0D4/wizard-falcon.png);
+  background-repeat: no-repeat;
+  background-size: 100px 95px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#nous-hermes-13b {
+  background: url(https://i.ibb.co/sm8VgtL/nous-hermes.png);
+  background-repeat: no-repeat;
+  background-size: 100px 95px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#airoboros-7b, #airoboros-13b {
+  background: url(https://i.ibb.co/NLchBkB/airoboros.png);
+  background-repeat: no-repeat;
+  background-size: 100px 95px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#samantha-7b, #samantha-13b, #samantha-33b {
+  background: url(https://i.ibb.co/72t5pyP/samantha.png);
+  background-repeat: no-repeat;
+  background-size: 100px 95px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#lazarus-30b {
+  background: url(https://i.ibb.co/Zm2Bdzt/lazarus.png);
+  background-repeat: no-repeat;
+  background-size: 100px 95px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#chronos-13b, #chronos-33b {
+  background: url(https://i.ibb.co/sQZ3L8j/chronos.png);
+  background-repeat: no-repeat;
+  background-size: 100px 95px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#wizardlm-13b, #wizardlm-30b {
+  background: url(https://i.ibb.co/SRXWKz9/WizardLM.png);
+  background-repeat: no-repeat;
+  background-size: 100px 95px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#replit-3b {
+  background: url(https://i.ibb.co/BrKCKYq/replit.png);
+  background-repeat: no-repeat;
+  background-size: 100px 95px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#byom {
+  background: url(https://i.ibb.co/YhM4B2X/byom.png);
+  background-repeat: no-repeat;
+  background-size: 100px 95px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;
+}
+
+#chosen-model {
+  background: url(https://i.ibb.co/dLmNh2v/chosen.png);
+  background-repeat: no-repeat;
+  background-size: 100px 95px;
+  color: transparent;  
+  width: 100px;
+  height: 100px;
+  margin: auto;  
+}
+
+.sub-container > div {
+  min-width: max-content !important;
+}
+"""