""" The gradio demo server with multiple tabs. It supports chatting with a single model or chatting with two models side-by-side. """ import argparse import pickle import time import gradio as gr from fastchat.constants import ( SESSION_EXPIRATION_TIME, ) from fastchat.serve.gradio_block_arena_anony import ( build_side_by_side_ui_anony, load_demo_side_by_side_anony, set_global_vars_anony, ) from fastchat.serve.gradio_block_arena_named import ( build_side_by_side_ui_named, load_demo_side_by_side_named, set_global_vars_named, ) from fastchat.serve.gradio_web_server import ( set_global_vars, block_css, build_single_model_ui, build_about, get_model_list, load_demo_single, ip_expiration_dict, get_ip, ) from fastchat.serve.monitor.monitor import build_leaderboard_tab from fastchat.utils import ( build_logger, get_window_url_params_js, get_window_url_params_with_tos_js, parse_gradio_auth_creds, ) import pdb logger = build_logger("gradio_web_server_multi", "gradio_web_server_multi.log") def load_demo(url_params, request: gr.Request): logger.info("load_demo_multi") global models ip = get_ip(request) logger.info(f"load_demo. ip: {ip}. params: {url_params}") ip_expiration_dict[ip] = time.time() + SESSION_EXPIRATION_TIME selected = 0 if "arena" in url_params: selected = 0 elif "compare" in url_params: selected = 1 elif "single" in url_params: selected = 2 elif "leaderboard" in url_params: selected = 3 if args.model_list_mode == "reload": if args.anony_only_for_proprietary_model: models = get_model_list( args.controller_url, args.register_openai_compatible_models, False, False, False, ) else: models = get_model_list( args.controller_url, args.register_openai_compatible_models, args.add_chatgpt, args.add_claude, args.add_palm, ) single_updates = load_demo_single(models, url_params) models_anony = list(models) if args.anony_only_for_proprietary_model: # Only enable these models in anony battles. if args.add_chatgpt: models_anony += [ "gpt-4", "gpt-3.5-turbo", "gpt-4-turbo", "gpt-3.5-turbo-1106", ] if args.add_claude: models_anony += ["claude-2.1", "claude-2.0", "claude-1", "claude-instant-1"] if args.add_palm: models_anony += ["palm-2"] models_anony = list(set(models_anony)) side_by_side_anony_updates = load_demo_side_by_side_anony(models_anony, url_params) side_by_side_named_updates = load_demo_side_by_side_named(models, url_params) return ( (gr.Tabs.update(selected=selected),) + single_updates + side_by_side_anony_updates + side_by_side_named_updates ) def build_demo(models, elo_results_file, leaderboard_table_file): # text_size = gr.themes.sizes.text_md with gr.Blocks( title="Chat with Open Large Language Models", theme=gr.themes.Default(), css=block_css, ) as demo: logger.info("build demo") url_params = gr.JSON(visible=False) with gr.Tabs() as tabs: with gr.Tab("Arena (battle)", id=0): side_by_side_anony_list = build_side_by_side_ui_anony(models) with gr.Tab("Arena (side-by-side)", id=1): side_by_side_named_list = build_side_by_side_ui_named(models) with gr.Tab("Direct Chat", id=2): single_model_list = build_single_model_ui( models, add_promotion_links=True ) if elo_results_file: with gr.Tab("Leaderboard", id=3): build_leaderboard_tab(elo_results_file, leaderboard_table_file) with gr.Tab("About Us", id=4): about = build_about() logger.info(f"url_param: {url_params}") if args.model_list_mode not in ["once", "reload"]: raise ValueError(f"Unknown model list mode: {args.model_list_mode}") if args.show_terms_of_use: load_js = get_window_url_params_with_tos_js else: load_js = get_window_url_params_js logger.info(f"url_param: {url_params}") demo.load( load_demo, [url_params], [tabs] + single_model_list + side_by_side_anony_list + side_by_side_named_list, _js=load_js, ) # pdb.set_trace() logger.info("build demo end") return demo if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--host", type=str, default="127.0.0.1") parser.add_argument("--port", type=int) parser.add_argument( "--share", action="store_true", help="Whether to generate a public, shareable link", ) parser.add_argument( "--controller-url", type=str, default="http://172.17.8.138:21001", help="The address of the controller", ) parser.add_argument( "--concurrency-count", type=int, default=10, help="The concurrency count of the gradio queue", ) parser.add_argument( "--model-list-mode", type=str, default="once", choices=["once", "reload"], help="Whether to load the model list once or reload the model list every time.", ) parser.add_argument( "--moderate", action="store_true", help="Enable content moderation to block unsafe inputs", ) parser.add_argument( "--show-terms-of-use", action="store_true", help="Shows term of use before loading the demo", ) parser.add_argument( "--add-chatgpt", action="store_true", help="Add OpenAI's ChatGPT models (gpt-3.5-turbo, gpt-4)", ) parser.add_argument( "--add-claude", action="store_true", help="Add Anthropic's Claude models (claude-2, claude-instant-1)", ) parser.add_argument( "--add-palm", action="store_true", help="Add Google's PaLM model (PaLM 2 for Chat: chat-bison@001)", ) parser.add_argument( "--anony-only-for-proprietary-model", action="store_true", help="Only add ChatGPT, Claude, Bard under anony battle tab", ) parser.add_argument( "--register-openai-compatible-models", type=str, help="Register custom OpenAI API compatible models by loading them from a JSON file", ) parser.add_argument( "--gradio-auth-path", type=str, help='Set the gradio authentication file path. The file should contain one or more user:password pairs in this format: "u1:p1,u2:p2,u3:p3"', default=None, ) parser.add_argument( "--elo-results-file", type=str, help="Load leaderboard results and plots" ) parser.add_argument( "--leaderboard-table-file", type=str, help="Load leaderboard results and plots" ) args = parser.parse_args() logger.info(f"args: {args}") # Set global variables set_global_vars(args.controller_url, args.moderate) set_global_vars_named(args.moderate) set_global_vars_anony(args.moderate) if args.anony_only_for_proprietary_model: models = get_model_list( args.controller_url, args.register_openai_compatible_models, False, False, False, ) else: models = get_model_list( args.controller_url, args.register_openai_compatible_models, args.add_chatgpt, args.add_claude, args.add_palm, ) logger.info(f'models: {models}') # Set authorization credentials auth = None if args.gradio_auth_path is not None: auth = parse_gradio_auth_creds(args.gradio_auth_path) # Launch the demo demo = build_demo(models, args.elo_results_file, args.leaderboard_table_file) # demo.launch( # server_name=args.host, # server_port=args.port, # share=args.share, # max_threads=400, # auth=auth, # ) # demo.queue( # concurrency_count=args.concurrency_count, status_update_rate=10, api_open=False # ).launch( # server_name=args.host, # server_port=args.port, # share=args.share, # max_threads=400, # auth=auth, # ) demo.queue( status_update_rate=10, api_open=False ).launch( server_name=args.host, server_port=args.port, share=True, max_threads=400, auth=auth, )