import ast
import copy
import functools
import inspect
import itertools
import json
import os
import pprint
import random
import shutil
import sys
import time
import traceback
import uuid
import filelock
import numpy as np
import pandas as pd
import requests
from iterators import TimeoutIterator
from gradio_utils.css import get_css
from gradio_utils.prompt_form import make_chatbots, get_chatbot_name
from src.db_utils import set_userid, get_username_direct
from src.tts_utils import combine_audios
# This is a hack to prevent Gradio from phoning home when it gets imported
os.environ['GRADIO_ANALYTICS_ENABLED'] = 'False'
def my_get(url, **kwargs):
print('Gradio HTTP request redirected to localhost :)', flush=True)
kwargs.setdefault('allow_redirects', True)
return requests.api.request('get', 'http://127.0.0.1/', **kwargs)
original_get = requests.get
requests.get = my_get
import gradio as gr
requests.get = original_get
def fix_pydantic_duplicate_validators_error():
try:
from pydantic import class_validators
class_validators.in_ipython = lambda: True # type: ignore[attr-defined]
except ImportError:
pass
fix_pydantic_duplicate_validators_error()
from enums import DocumentSubset, no_model_str, no_lora_str, no_server_str, LangChainAction, LangChainMode, \
DocumentChoice, langchain_modes_intrinsic, LangChainTypes, langchain_modes_non_db, gr_to_lg, invalid_key_msg, \
LangChainAgent, docs_ordering_types, docs_token_handlings, docs_joiner_default
from gradio_themes import H2oTheme, SoftTheme, get_h2o_title, get_simple_title, \
get_dark_js, get_heap_js, wrap_js_to_lambda, \
spacing_xsm, radius_xsm, text_xsm
from prompter import prompt_type_to_model_name, prompt_types_strings, inv_prompt_type_to_model_lower, non_hf_types, \
get_prompt, model_names_curated, get_system_prompts, get_llava_prompts
from utils import flatten_list, zip_data, s3up, clear_torch_cache, get_torch_allocated, system_info_print, \
ping, makedirs, get_kwargs, system_info, ping_gpu, get_url, get_local_ip, \
save_generate_output, url_alive, remove, dict_to_html, text_to_html, lg_to_gr, str_to_dict, have_serpapi, \
have_librosa, have_gradio_pdf, have_pyrubberband, is_gradio_version4, have_fiftyone, n_gpus_global, \
_save_generate_tokens, get_accordion_named
from gen import get_model, languages_covered, evaluate, score_qa, inputs_kwargs_list, \
get_max_max_new_tokens, get_minmax_top_k_docs, history_to_context, langchain_actions, langchain_agents_list, \
evaluate_fake, merge_chat_conversation_history, switch_a_roo_llama, get_model_max_length_from_tokenizer, \
get_model_retry, remove_refs, get_on_disk_models, get_llama_lower_hf, model_name_to_prompt_type
from evaluate_params import eval_func_param_names, no_default_param_names, eval_func_param_names_defaults, \
input_args_list, key_overrides
from apscheduler.schedulers.background import BackgroundScheduler
def fix_text_for_gradio(text, fix_new_lines=False, fix_latex_dollars=True):
if isinstance(text, tuple):
# images, audio, etc.
return text
if not isinstance(text, str):
# e.g. list for extraction
text = str(text)
if fix_latex_dollars:
ts = text.split('```')
for parti, part in enumerate(ts):
inside = parti % 2 == 1
if not inside:
ts[parti] = ts[parti].replace('$', 'īšŠ')
text = '```'.join(ts)
if fix_new_lines:
# let Gradio handle code, since got improved recently
## FIXME: below conflicts with Gradio, but need to see if can handle multiple \n\n\n etc. properly as is.
# ensure good visually, else markdown ignores multiple \n
# handle code blocks
ts = text.split('```')
for parti, part in enumerate(ts):
inside = parti % 2 == 1
if not inside:
ts[parti] = ts[parti].replace('\n', '
')
text = '```'.join(ts)
return text
def is_from_ui(requests_state1):
return isinstance(requests_state1, dict) and 'username' in requests_state1 and requests_state1['username']
def is_valid_key(enforce_h2ogpt_api_key, enforce_h2ogpt_ui_key, h2ogpt_api_keys, h2ogpt_key1, requests_state1=None):
from_ui = is_from_ui(requests_state1)
if from_ui and not enforce_h2ogpt_ui_key:
# no token barrier
return 'not enforced'
elif not from_ui and not enforce_h2ogpt_api_key:
# no token barrier
return 'not enforced'
else:
valid_key = False
if isinstance(h2ogpt_api_keys, list) and h2ogpt_key1 in h2ogpt_api_keys:
# passed token barrier
valid_key = True
elif isinstance(h2ogpt_api_keys, str) and os.path.isfile(h2ogpt_api_keys):
with filelock.FileLock(h2ogpt_api_keys + '.lock'):
with open(h2ogpt_api_keys, 'rt') as f:
h2ogpt_api_keys = json.load(f)
if h2ogpt_key1 in h2ogpt_api_keys:
valid_key = True
return valid_key
def get_one_key(h2ogpt_api_keys, enforce_h2ogpt_api_key):
if not enforce_h2ogpt_api_key:
# return None so OpenAI server has no keyed access if not enforcing API key on h2oGPT regardless if keys passed
return None
if isinstance(h2ogpt_api_keys, list) and h2ogpt_api_keys:
return h2ogpt_api_keys[0]
elif isinstance(h2ogpt_api_keys, str) and os.path.isfile(h2ogpt_api_keys):
with filelock.FileLock(h2ogpt_api_keys + '.lock'):
with open(h2ogpt_api_keys, 'rt') as f:
h2ogpt_api_keys = json.load(f)
if h2ogpt_api_keys:
return h2ogpt_api_keys[0]
def get_prompt_type1(is_public, **kwargs):
prompt_types_strings_used = prompt_types_strings.copy()
if kwargs['model_lock']:
prompt_types_strings_used += [no_model_str]
default_prompt_type = kwargs['prompt_type'] or no_model_str
else:
default_prompt_type = kwargs['prompt_type'] or 'plain'
prompt_type = gr.Dropdown(prompt_types_strings_used,
value=default_prompt_type,
label="Choose/Select Prompt Type",
info="Auto-Detected if known (plain means failed to detect)",
visible=not kwargs['model_lock'],
interactive=not is_public,
)
return prompt_type
def get_prompt_type2(is_public, **kwargs):
prompt_types_strings_used = prompt_types_strings.copy()
if kwargs['model_lock']:
prompt_types_strings_used += [no_model_str]
default_prompt_type = kwargs['prompt_type'] or no_model_str
else:
default_prompt_type = kwargs['prompt_type'] or 'plain'
prompt_type2 = gr.Dropdown(prompt_types_strings_used,
value=default_prompt_type,
label="Choose/Select Prompt Type Model 2",
info="Auto-Detected if known (plain means failed to detect)",
visible=False and not kwargs['model_lock'],
interactive=not is_public)
return prompt_type2
def go_gradio(**kwargs):
page_title = kwargs['page_title']
allow_api = kwargs['allow_api']
is_public = kwargs['is_public']
is_hf = kwargs['is_hf']
memory_restriction_level = kwargs['memory_restriction_level']
n_gpus = kwargs['n_gpus']
admin_pass = kwargs['admin_pass']
model_states = kwargs['model_states']
dbs = kwargs['dbs']
db_type = kwargs['db_type']
visible_langchain_actions = kwargs['visible_langchain_actions']
visible_langchain_agents = kwargs['visible_langchain_agents']
allow_upload_to_user_data = kwargs['allow_upload_to_user_data']
allow_upload_to_my_data = kwargs['allow_upload_to_my_data']
enable_sources_list = kwargs['enable_sources_list']
enable_url_upload = kwargs['enable_url_upload']
enable_text_upload = kwargs['enable_text_upload']
use_openai_embedding = kwargs['use_openai_embedding']
hf_embedding_model = kwargs['hf_embedding_model']
load_db_if_exists = kwargs['load_db_if_exists']
migrate_embedding_model = kwargs['migrate_embedding_model']
auto_migrate_db = kwargs['auto_migrate_db']
captions_model = kwargs['captions_model']
caption_loader = kwargs['caption_loader']
doctr_loader = kwargs['doctr_loader']
llava_model = kwargs['llava_model']
asr_model = kwargs['asr_model']
asr_loader = kwargs['asr_loader']
n_jobs = kwargs['n_jobs']
verbose = kwargs['verbose']
# for dynamic state per user session in gradio
model_state0 = kwargs['model_state0']
score_model_state0 = kwargs['score_model_state0']
my_db_state0 = kwargs['my_db_state0']
selection_docs_state0 = kwargs['selection_docs_state0']
visible_models_state0 = kwargs['visible_models_state0']
roles_state0 = kwargs['roles_state0']
# For Heap analytics
is_heap_analytics_enabled = kwargs['enable_heap_analytics']
heap_app_id = kwargs['heap_app_id']
# easy update of kwargs needed for evaluate() etc.
queue = True
allow_upload = allow_upload_to_user_data or allow_upload_to_my_data
allow_upload_api = allow_api and allow_upload
kwargs.update(locals())
# import control
if kwargs['langchain_mode'] != 'Disabled':
from gpt_langchain import file_types, have_arxiv
else:
have_arxiv = False
file_types = []
if 'mbart-' in kwargs['model_lower']:
instruction_label_nochat = "Text to translate"
else:
instruction_label_nochat = "Instruction (Shift-Enter or push Submit to send message," \
" use Enter for multiple input lines)"
title = 'h2oGPT'
if kwargs['visible_h2ogpt_links']:
description = """h2oGPT LLM Leaderboard LLM Studio
CodeLlama
đ¤ Models
h2oGPTe"""
else:
description = None
description_bottom = "If this host is busy, try
[Multi-Model](https://gpt.h2o.ai)
[CodeLlama](https://codellama.h2o.ai)
[Llama2 70B](https://llama.h2o.ai)
[Falcon 40B](https://falcon.h2o.ai)
[HF Spaces1](https://huggingface.co/spaces/h2oai/h2ogpt-chatbot)
[HF Spaces2](https://huggingface.co/spaces/h2oai/h2ogpt-chatbot2)
"
if is_hf:
description_bottom += ''''''
task_info_md = ''
css_code = get_css(kwargs)
if kwargs['gradio_offline_level'] >= 0:
# avoid GoogleFont that pulls from internet
if kwargs['gradio_offline_level'] == 1:
# front end would still have to download fonts or have cached it at some point
base_font = 'Source Sans Pro'
else:
base_font = 'Helvetica'
theme_kwargs = dict(font=(base_font, 'ui-sans-serif', 'system-ui', 'sans-serif'),
font_mono=('IBM Plex Mono', 'ui-monospace', 'Consolas', 'monospace'))
else:
theme_kwargs = dict()
if kwargs['gradio_size'] == 'xsmall':
theme_kwargs.update(dict(spacing_size=spacing_xsm, text_size=text_xsm, radius_size=radius_xsm))
elif kwargs['gradio_size'] in [None, 'small']:
theme_kwargs.update(dict(spacing_size=gr.themes.sizes.spacing_sm, text_size=gr.themes.sizes.text_sm,
radius_size=gr.themes.sizes.spacing_sm))
elif kwargs['gradio_size'] == 'large':
theme_kwargs.update(dict(spacing_size=gr.themes.sizes.spacing_lg, text_size=gr.themes.sizes.text_lg),
radius_size=gr.themes.sizes.spacing_lg)
elif kwargs['gradio_size'] == 'medium':
theme_kwargs.update(dict(spacing_size=gr.themes.sizes.spacing_md, text_size=gr.themes.sizes.text_md,
radius_size=gr.themes.sizes.spacing_md))
theme = H2oTheme(**theme_kwargs) if kwargs['h2ocolors'] else SoftTheme(**theme_kwargs)
demo = gr.Blocks(theme=theme, css=css_code, title=page_title, analytics_enabled=False)
callback = gr.CSVLogger()
# modify, if model lock then don't show models, then need prompts in expert
kwargs['visible_models_tab'] = kwargs['visible_models_tab'] and not bool(kwargs['model_lock'])
# Initial model options
if kwargs['visible_all_prompter_models']:
model_options0 = flatten_list(list(prompt_type_to_model_name.values())) + kwargs['extra_model_options']
else:
model_options0 = []
if kwargs['visible_curated_models']:
model_options0.extend(model_names_curated)
model_options0.extend(kwargs['extra_model_options'])
if kwargs['base_model'].strip() and kwargs['base_model'].strip() not in model_options0:
model_options0 = [kwargs['base_model'].strip()] + model_options0
if kwargs['add_disk_models_to_ui'] and kwargs['visible_models_tab'] and not kwargs['model_lock']:
model_options0.extend(get_on_disk_models(llamacpp_path=kwargs['llamacpp_path'],
use_auth_token=kwargs['use_auth_token'],
trust_remote_code=kwargs['trust_remote_code']))
model_options0 = sorted(set(model_options0))
# Initial LORA options
lora_options = kwargs['extra_lora_options']
if kwargs['lora_weights'].strip() and kwargs['lora_weights'].strip() not in lora_options:
lora_options = [kwargs['lora_weights'].strip()] + lora_options
# Initial server options
server_options = kwargs['extra_server_options']
if kwargs['inference_server'].strip() and kwargs['inference_server'].strip() not in server_options:
server_options = [kwargs['inference_server'].strip()] + server_options
if os.getenv('OPENAI_API_KEY'):
if 'openai_chat' not in server_options:
server_options += ['openai_chat']
if 'openai' not in server_options:
server_options += ['openai']
# always add in no lora case
# add fake space so doesn't go away in gradio dropdown
model_options0 = [no_model_str] + sorted(model_options0)
lora_options = [no_lora_str] + sorted(lora_options)
server_options = [no_server_str] + sorted(server_options)
# always add in no model case so can free memory
# add fake space so doesn't go away in gradio dropdown
# transcribe, will be detranscribed before use by evaluate()
if not kwargs['base_model'].strip():
kwargs['base_model'] = no_model_str
if not kwargs['lora_weights'].strip():
kwargs['lora_weights'] = no_lora_str
if not kwargs['inference_server'].strip():
kwargs['inference_server'] = no_server_str
# transcribe for gradio
kwargs['gpu_id'] = str(kwargs['gpu_id'])
no_model_msg = 'h2oGPT [ !!! Please Load Model in Models Tab !!! ]'
chat_name0 = get_chatbot_name(kwargs.get("base_model"), kwargs.get("llamacpp_dict", {}).get("model_path_llama"))
output_label0 = chat_name0 if kwargs.get('base_model') else no_model_msg
output_label0_model2 = no_model_msg
def update_prompt(prompt_type1, prompt_dict1, model_state1, which_model=0, global_scope=False):
if not prompt_type1 or which_model != 0:
# keep prompt_type and prompt_dict in sync if possible
prompt_type1 = kwargs.get('prompt_type', prompt_type1)
prompt_dict1 = kwargs.get('prompt_dict', prompt_dict1)
# prefer model specific prompt type instead of global one
if not global_scope:
if not prompt_type1 or which_model != 0:
prompt_type1 = model_state1.get('prompt_type', prompt_type1)
prompt_dict1 = model_state1.get('prompt_dict', prompt_dict1)
if not prompt_dict1 or which_model != 0:
# if still not defined, try to get
prompt_dict1 = kwargs.get('prompt_dict', prompt_dict1)
if not global_scope:
if not prompt_dict1 or which_model != 0:
prompt_dict1 = model_state1.get('prompt_dict', prompt_dict1)
if not global_scope and not prompt_type1:
# if still not defined, use plain
prompt_type1 = 'plain'
return prompt_type1, prompt_dict1
def visible_models_to_model_choice(visible_models1, api=False):
if isinstance(visible_models1, list):
assert len(
visible_models1) >= 1, "Invalid visible_models1=%s, can only be single entry" % visible_models1
# just take first
model_active_choice1 = visible_models1[0]
elif isinstance(visible_models1, (str, int)):
model_active_choice1 = visible_models1
else:
assert isinstance(visible_models1, type(None)), "Invalid visible_models1=%s" % visible_models1
model_active_choice1 = visible_models1
if model_active_choice1 is not None:
if isinstance(model_active_choice1, str):
base_model_list = [
x['base_model'] if x['base_model'] != 'llama' or not x.get("llamacpp_dict", {}).get(
'model_path_llama', '') else x.get("llamacpp_dict", {})[
'model_path_llama'] for x in model_states]
if model_active_choice1 in base_model_list:
# if dups, will just be first one
model_active_choice1 = base_model_list.index(model_active_choice1)
else:
# NOTE: Could raise, but sometimes raising in certain places fails too hard and requires UI restart
if api:
raise ValueError(
"Invalid model %s, valid models are: %s" % (model_active_choice1, base_model_list))
model_active_choice1 = 0
else:
model_active_choice1 = 0
return model_active_choice1
default_kwargs = {k: kwargs[k] for k in eval_func_param_names_defaults}
# ensure prompt_type consistent with prep_bot(), so nochat API works same way
default_kwargs['prompt_type'], default_kwargs['prompt_dict'] = \
update_prompt(default_kwargs['prompt_type'], default_kwargs['prompt_dict'],
model_state1=model_state0,
which_model=visible_models_to_model_choice(kwargs['visible_models']),
global_scope=True, # don't assume state0 is the prompt for all models
)
for k in no_default_param_names:
default_kwargs[k] = ''
def dummy_fun(x):
# need dummy function to block new input from being sent until output is done,
# else gets input_list at time of submit that is old, and shows up as truncated in chatbot
return x
def update_auth_selection(auth_user, selection_docs_state1, save=False):
# in-place update of both
if 'selection_docs_state' not in auth_user:
auth_user['selection_docs_state'] = selection_docs_state0
for k, v in auth_user['selection_docs_state'].items():
if isinstance(selection_docs_state1[k], dict):
if save:
auth_user['selection_docs_state'][k].clear()
auth_user['selection_docs_state'][k].update(selection_docs_state1[k])
else:
selection_docs_state1[k].clear()
selection_docs_state1[k].update(auth_user['selection_docs_state'][k])
elif isinstance(selection_docs_state1[k], list):
if save:
auth_user['selection_docs_state'][k].clear()
auth_user['selection_docs_state'][k].extend(selection_docs_state1[k])
else:
selection_docs_state1[k].clear()
selection_docs_state1[k].extend(auth_user['selection_docs_state'][k])
else:
raise RuntimeError("Bad type: %s" % selection_docs_state1[k])
# BEGIN AUTH THINGS
def auth_func(username1, password1, auth_pairs=None, auth_filename=None,
auth_access=None,
auth_freeze=None,
guest_name=None,
selection_docs_state1=None,
selection_docs_state00=None,
**kwargs):
assert auth_freeze is not None
if selection_docs_state1 is None:
selection_docs_state1 = selection_docs_state00
assert selection_docs_state1 is not None
assert auth_filename and isinstance(auth_filename, str), "Auth file must be a non-empty string, got: %s" % str(
auth_filename)
if auth_access == 'open' and username1 == guest_name:
return True
if username1 == '':
# some issue with login
return False
with filelock.FileLock(auth_filename + '.lock'):
auth_dict = {}
if os.path.isfile(auth_filename):
try:
with open(auth_filename, 'rt') as f:
auth_dict = json.load(f)
except json.decoder.JSONDecodeError as e:
print("Auth exception: %s" % str(e), flush=True)
shutil.move(auth_filename, auth_filename + '.bak' + str(uuid.uuid4()))
auth_dict = {}
if username1 in auth_dict and username1 in auth_pairs:
if password1 == auth_dict[username1]['password'] and password1 == auth_pairs[username1]:
auth_user = auth_dict[username1]
update_auth_selection(auth_user, selection_docs_state1)
save_auth_dict(auth_dict, auth_filename)
return True
else:
return False
elif username1 in auth_dict:
if password1 == auth_dict[username1]['password']:
auth_user = auth_dict[username1]
update_auth_selection(auth_user, selection_docs_state1)
save_auth_dict(auth_dict, auth_filename)
return True
else:
return False
elif username1 in auth_pairs:
# copy over CLI auth to file so only one state to manage
auth_dict[username1] = dict(password=auth_pairs[username1], userid=str(uuid.uuid4()))
auth_user = auth_dict[username1]
update_auth_selection(auth_user, selection_docs_state1)
save_auth_dict(auth_dict, auth_filename)
return True
else:
if auth_access == 'closed':
return False
# open access
auth_dict[username1] = dict(password=password1, userid=str(uuid.uuid4()))
auth_user = auth_dict[username1]
update_auth_selection(auth_user, selection_docs_state1)
save_auth_dict(auth_dict, auth_filename)
if auth_access == 'open':
return True
else:
raise RuntimeError("Invalid auth_access: %s" % auth_access)
def auth_func_open(*args, **kwargs):
return True
def get_username(requests_state1):
username1 = None
if 'username' in requests_state1:
username1 = requests_state1['username']
return username1
def get_userid_auth_func(requests_state1, auth_filename=None, auth_access=None, guest_name=None, id0=None,
**kwargs):
if auth_filename and isinstance(auth_filename, str):
username1 = get_username(requests_state1)
if username1:
if username1 == guest_name:
return str(uuid.uuid4())
with filelock.FileLock(auth_filename + '.lock'):
if os.path.isfile(auth_filename):
with open(auth_filename, 'rt') as f:
auth_dict = json.load(f)
if username1 in auth_dict:
return auth_dict[username1]['userid']
# if here, then not persistently associated with username1,
# but should only be one-time asked if going to persist within a single session!
return id0 or str(uuid.uuid4())
get_userid_auth = functools.partial(get_userid_auth_func,
auth_filename=kwargs['auth_filename'],
auth_access=kwargs['auth_access'],
guest_name=kwargs['guest_name'],
)
if kwargs['auth_access'] == 'closed':
auth_message1 = "Closed access"
else:
auth_message1 = "WELCOME! Open access" \
" (%s/%s or any unique user/pass)" % (kwargs['guest_name'], kwargs['guest_name'])
if kwargs['auth_message'] is not None:
auth_message = kwargs['auth_message']
else:
auth_message = auth_message1
# always use same callable
auth_pairs0 = {}
if isinstance(kwargs['auth'], list):
for k, v in kwargs['auth']:
auth_pairs0[k] = v
authf = functools.partial(auth_func,
auth_pairs=auth_pairs0,
auth_filename=kwargs['auth_filename'],
auth_access=kwargs['auth_access'],
auth_freeze=kwargs['auth_freeze'],
guest_name=kwargs['guest_name'],
selection_docs_state00=copy.deepcopy(selection_docs_state0))
def get_request_state(requests_state1, request, db1s):
# if need to get state, do it now
if not requests_state1:
requests_state1 = requests_state0.copy()
if requests:
if not requests_state1.get('headers', '') and hasattr(request, 'headers'):
requests_state1.update(request.headers)
if not requests_state1.get('host', '') and hasattr(request, 'host'):
requests_state1.update(dict(host=request.host))
if not requests_state1.get('host2', '') and hasattr(request, 'client') and hasattr(request.client, 'host'):
requests_state1.update(dict(host2=request.client.host))
if not requests_state1.get('username', '') and hasattr(request, 'username'):
# use already-defined username instead of keep changing to new uuid
# should be same as in requests_state1
db_username = get_username_direct(db1s)
requests_state1.update(dict(username=request.username or db_username or str(uuid.uuid4())))
requests_state1 = {str(k): str(v) for k, v in requests_state1.items()}
return requests_state1
def user_state_setup(db1s, requests_state1, request: gr.Request, *args):
requests_state1 = get_request_state(requests_state1, request, db1s)
set_userid(db1s, requests_state1, get_userid_auth)
args_list = [db1s, requests_state1] + list(args)
return tuple(args_list)
# END AUTH THINGS
def allow_empty_instruction(langchain_mode1, document_subset1, langchain_action1):
allow = False
allow |= langchain_action1 not in [LangChainAction.QUERY.value,
LangChainAction.IMAGE_QUERY.value,
LangChainAction.IMAGE_CHANGE.value,
LangChainAction.IMAGE_GENERATE.value,
LangChainAction.IMAGE_GENERATE_HIGH.value,
]
allow |= document_subset1 in [DocumentSubset.TopKSources.name]
if langchain_mode1 in [LangChainMode.LLM.value]:
allow = False
return allow
image_audio_loaders_options0, image_audio_loaders_options, \
pdf_loaders_options0, pdf_loaders_options, \
url_loaders_options0, url_loaders_options = lg_to_gr(**kwargs)
jq_schema0 = '.[]'
def click_js():
return """function audioRecord() {
var xPathRes = document.evaluate ('//*[contains(@class, "record")]', document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
xPathRes.singleNodeValue.click();}"""
def click_submit():
return """function check() {
document.getElementById("submit").click();
}"""
def click_stop():
return """function check() {
document.getElementById("stop").click();
}"""
if is_gradio_version4:
noqueue_kwargs = dict(concurrency_limit=None)
noqueue_kwargs2 = dict(concurrency_limit=None)
mic_kwargs = dict(js=click_js())
submit_kwargs = dict(js=click_submit())
stop_kwargs = dict(js=click_stop())
dark_kwargs = dict(js=wrap_js_to_lambda(0, get_dark_js()))
queue_kwargs = dict(default_concurrency_limit=kwargs['concurrency_count'])
mic_sources_kwargs = dict(sources=['microphone'],
waveform_options=dict(show_controls=False, show_recording_waveform=False))
else:
noqueue_kwargs = dict(queue=False)
noqueue_kwargs2 = dict()
mic_kwargs = dict(_js=click_js())
submit_kwargs = dict(_js=click_submit())
stop_kwargs = dict(_js=click_stop())
dark_kwargs = dict(_js=wrap_js_to_lambda(0, get_dark_js()))
queue_kwargs = dict(concurrency_count=kwargs['concurrency_count'])
mic_sources_kwargs = dict(source='microphone')
with demo:
# avoid actual model/tokenizer here or anything that would be bad to deepcopy
# https://github.com/gradio-app/gradio/issues/3558
model_state = gr.State(
dict(model='model', tokenizer='tokenizer', device=kwargs['device'],
base_model=kwargs['base_model'],
tokenizer_base_model=kwargs['tokenizer_base_model'],
lora_weights=kwargs['lora_weights'],
inference_server=kwargs['inference_server'],
prompt_type=kwargs['prompt_type'],
prompt_dict=kwargs['prompt_dict'],
visible_models=visible_models_to_model_choice(kwargs['visible_models']),
h2ogpt_key=None, # only apply at runtime when doing API call with gradio inference server
)
)
def update_langchain_mode_paths(selection_docs_state1):
dup = selection_docs_state1['langchain_mode_paths'].copy()
for k, v in dup.items():
if k not in selection_docs_state1['langchain_modes']:
selection_docs_state1['langchain_mode_paths'].pop(k)
for k in selection_docs_state1['langchain_modes']:
if k not in selection_docs_state1['langchain_mode_types']:
# if didn't specify shared, then assume scratch if didn't login or personal if logged in
selection_docs_state1['langchain_mode_types'][k] = LangChainTypes.PERSONAL.value
return selection_docs_state1
# Setup some gradio states for per-user dynamic state
model_state2 = gr.State(kwargs['model_state_none'].copy())
model_options_state = gr.State([model_options0])
lora_options_state = gr.State([lora_options])
server_options_state = gr.State([server_options])
my_db_state = gr.State(my_db_state0)
chat_state = gr.State({})
if kwargs['enable_tts'] and kwargs['tts_model'].startswith('tts_models/'):
from src.tts_coqui import get_role_to_wave_map
roles_state = gr.State(roles_state0 if roles_state0 else get_role_to_wave_map())
else:
roles_state = gr.State({})
docs_state00 = kwargs['document_choice'] + [DocumentChoice.ALL.value]
docs_state0 = []
[docs_state0.append(x) for x in docs_state00 if x not in docs_state0]
docs_state = gr.State(docs_state0)
viewable_docs_state0 = ['None']
viewable_docs_state = gr.State(viewable_docs_state0)
selection_docs_state0 = update_langchain_mode_paths(selection_docs_state0)
selection_docs_state = gr.State(selection_docs_state0)
requests_state0 = dict(headers='', host='', username='')
requests_state = gr.State(requests_state0)
if kwargs['visible_h2ogpt_logo']:
if description is None:
description = ''
gr.Markdown(f"""
{get_h2o_title(title, description, visible_h2ogpt_qrcode=kwargs['visible_h2ogpt_qrcode'])
if kwargs['h2ocolors'] else get_simple_title(title, description)}
""")
# go button visible if
base_wanted = kwargs['base_model'] != no_model_str and kwargs['login_mode_if_model0']
go_btn = gr.Button(value="ENTER", visible=base_wanted, variant="primary")
nas = ' '.join(['NA'] * len(kwargs['model_states']))
res_value = "Response Score: NA" if not kwargs[
'model_lock'] else "Response Scores: %s" % nas
user_can_do_sum = kwargs['langchain_mode'] != LangChainMode.DISABLED.value and \
(kwargs['visible_side_bar'] or kwargs['visible_system_tab'])
if user_can_do_sum:
extra_prompt_form = ". Just Click Submit for simple Summarize/Extract"
else:
extra_prompt_form = ""
if allow_upload:
extra_prompt_form += ". Clicking Ingest adds text as URL/ArXiv/YouTube/Text."
if kwargs['input_lines'] > 1:
instruction_label = "Shift-Enter to Submit, Enter adds lines%s" % extra_prompt_form
else:
instruction_label = "Enter to Submit, Shift-Enter adds lines%s" % extra_prompt_form
def get_langchain_choices(selection_docs_state1):
langchain_modes = selection_docs_state1['langchain_modes']
if is_hf:
# don't show 'wiki' since only usually useful for internal testing at moment
no_show_modes = ['Disabled', 'wiki']
else:
no_show_modes = ['Disabled']
allowed_modes = langchain_modes.copy()
# allowed_modes = [x for x in allowed_modes if x in dbs]
allowed_modes += ['LLM']
if allow_upload_to_my_data and 'MyData' not in allowed_modes:
allowed_modes += ['MyData']
if allow_upload_to_user_data and 'UserData' not in allowed_modes:
allowed_modes += ['UserData']
choices = [x for x in langchain_modes if x in allowed_modes and x not in no_show_modes]
return choices
def get_df_langchain_mode_paths(selection_docs_state1, db1s, dbs1=None):
langchain_choices1 = get_langchain_choices(selection_docs_state1)
langchain_mode_paths = selection_docs_state1['langchain_mode_paths']
langchain_mode_paths = {k: v for k, v in langchain_mode_paths.items() if k in langchain_choices1}
if langchain_mode_paths:
langchain_mode_paths = langchain_mode_paths.copy()
for langchain_mode1 in langchain_modes_non_db:
langchain_mode_paths.pop(langchain_mode1, None)
df1 = pd.DataFrame.from_dict(langchain_mode_paths.items(), orient='columns')
df1.columns = ['Collection', 'Path']
df1 = df1.set_index('Collection')
else:
df1 = pd.DataFrame(None)
langchain_mode_types = selection_docs_state1['langchain_mode_types']
langchain_mode_types = {k: v for k, v in langchain_mode_types.items() if k in langchain_choices1}
if langchain_mode_types:
langchain_mode_types = langchain_mode_types.copy()
for langchain_mode1 in langchain_modes_non_db:
langchain_mode_types.pop(langchain_mode1, None)
df2 = pd.DataFrame.from_dict(langchain_mode_types.items(), orient='columns')
df2.columns = ['Collection', 'Type']
df2 = df2.set_index('Collection')
from src.gpt_langchain import get_persist_directory, load_embed
persist_directory_dict = {}
embed_dict = {}
chroma_version_dict = {}
for langchain_mode3 in langchain_mode_types:
langchain_type3 = langchain_mode_types.get(langchain_mode3, LangChainTypes.EITHER.value)
persist_directory3, langchain_type3 = get_persist_directory(langchain_mode3,
langchain_type=langchain_type3,
db1s=db1s, dbs=dbs1)
got_embedding3, use_openai_embedding3, hf_embedding_model3 = load_embed(
persist_directory=persist_directory3)
persist_directory_dict[langchain_mode3] = persist_directory3
embed_dict[langchain_mode3] = 'OpenAI' if not hf_embedding_model3 else hf_embedding_model3
if os.path.isfile(os.path.join(persist_directory3, 'chroma.sqlite3')):
chroma_version_dict[langchain_mode3] = 'ChromaDB>=0.4'
elif os.path.isdir(os.path.join(persist_directory3, 'index')):
chroma_version_dict[langchain_mode3] = 'ChromaDB<0.4'
elif not os.listdir(persist_directory3):
if db_type == 'chroma':
chroma_version_dict[langchain_mode3] = 'ChromaDB>=0.4' # will be
elif db_type == 'chroma_old':
chroma_version_dict[langchain_mode3] = 'ChromaDB<0.4' # will be
else:
chroma_version_dict[langchain_mode3] = 'Weaviate' # will be
if isinstance(hf_embedding_model, dict):
hf_embedding_model3 = hf_embedding_model['name']
else:
hf_embedding_model3 = hf_embedding_model
assert isinstance(hf_embedding_model3, str)
embed_dict[langchain_mode3] = hf_embedding_model3 # will be
else:
chroma_version_dict[langchain_mode3] = 'Weaviate'
df3 = pd.DataFrame.from_dict(persist_directory_dict.items(), orient='columns')
df3.columns = ['Collection', 'Directory']
df3 = df3.set_index('Collection')
df4 = pd.DataFrame.from_dict(embed_dict.items(), orient='columns')
df4.columns = ['Collection', 'Embedding']
df4 = df4.set_index('Collection')
df5 = pd.DataFrame.from_dict(chroma_version_dict.items(), orient='columns')
df5.columns = ['Collection', 'DB']
df5 = df5.set_index('Collection')
else:
df2 = pd.DataFrame(None)
df3 = pd.DataFrame(None)
df4 = pd.DataFrame(None)
df5 = pd.DataFrame(None)
df_list = [df2, df1, df3, df4, df5]
df_list = [x for x in df_list if x.shape[1] > 0]
if len(df_list) > 1:
df = df_list[0].join(df_list[1:]).replace(np.nan, '').reset_index()
elif len(df_list) == 0:
df = df_list[0].replace(np.nan, '').reset_index()
else:
df = pd.DataFrame(None)
return df
normal_block = gr.Row(visible=not base_wanted, equal_height=False, elem_id="col_container")
with normal_block:
side_bar = gr.Column(elem_id="sidebar", scale=1, min_width=100, visible=kwargs['visible_side_bar'])
with side_bar:
with gr.Accordion("Chats", open=False, visible=True):
radio_chats = gr.Radio(value=None, label="Saved Chats", show_label=False,
visible=True, interactive=True,
type='value')
visible_speak_me = kwargs['enable_tts'] and kwargs['predict_from_text_func'] is not None
speak_human_button = gr.Button("Speak Instruction", visible=visible_speak_me, size='sm')
speak_bot_button = gr.Button("Speak Response", visible=visible_speak_me, size='sm')
stop_speak_button = gr.Button("Stop/Clear Speak", visible=visible_speak_me, size='sm')
if kwargs['enable_tts'] and kwargs['tts_model'].startswith('tts_models/'):
from src.tts_coqui import get_roles
chatbot_role = get_roles(choices=list(roles_state.value.keys()), value=kwargs['chatbot_role'])
else:
chatbot_role = gr.Dropdown(choices=['None'], visible=False, value='None')
if kwargs['enable_tts'] and kwargs['tts_model'].startswith('microsoft'):
from src.tts import get_speakers_gr
speaker = get_speakers_gr(value=kwargs['speaker'])
else:
speaker = gr.Radio(visible=False)
min_tts_speed = 1.0 if not have_pyrubberband else 0.1
tts_speed = gr.Number(minimum=min_tts_speed, maximum=10.0, step=0.1,
value=kwargs['tts_speed'],
label='Speech Speed',
visible=kwargs['enable_tts'] and not is_public,
interactive=not is_public)
upload_visible = kwargs['langchain_mode'] != 'Disabled' and allow_upload
url_visible = kwargs['langchain_mode'] != 'Disabled' and allow_upload and enable_url_upload
if have_arxiv and have_librosa:
url_label = 'URLs/ArXiv/Youtube'
elif have_arxiv:
url_label = 'URLs/ArXiv'
elif have_librosa:
url_label = 'URLs/Youtube'
else:
url_label = 'URLs'
text_visible = kwargs['langchain_mode'] != 'Disabled' and allow_upload and enable_text_upload
fileup_output_text = gr.Textbox(visible=False)
with gr.Accordion("Upload", open=False, visible=upload_visible and kwargs['actions_in_sidebar']):
fileup_output = gr.File(show_label=False,
file_types=['.' + x for x in file_types],
# file_types=['*', '*.*'], # for iPhone etc. needs to be unconstrained else doesn't work with extension-based restrictions
file_count="multiple",
scale=1,
min_width=0,
elem_id="warning", elem_classes="feedback",
)
if kwargs['actions_in_sidebar']:
max_quality = gr.Checkbox(label="Max Ingest Quality", value=kwargs['max_quality'],
visible=not is_public)
gradio_upload_to_chatbot = gr.Checkbox(label="Add Doc to Chat",
value=kwargs['gradio_upload_to_chatbot'])
url_text = gr.Textbox(label=url_label,
# placeholder="Enter Submits",
max_lines=1,
interactive=True,
visible=kwargs['actions_in_sidebar'])
user_text_text = gr.Textbox(label='Paste Text',
# placeholder="Enter Submits",
interactive=True,
visible=text_visible and kwargs['actions_in_sidebar'])
database_visible = kwargs['langchain_mode'] != 'Disabled'
langchain_choices0 = get_langchain_choices(selection_docs_state0)
serp_visible = os.environ.get('SERPAPI_API_KEY') is not None and have_serpapi
allowed_actions = [x for x in langchain_actions if x in visible_langchain_actions]
default_action = allowed_actions[0] if len(allowed_actions) > 0 else None
if not kwargs['actions_in_sidebar']:
max_quality = gr.Checkbox(label="Max Ingest Quality",
value=kwargs['max_quality'],
visible=not is_public)
gradio_upload_to_chatbot = gr.Checkbox(label="Add Doc to Chat",
value=kwargs['gradio_upload_to_chatbot'])
if not kwargs['actions_in_sidebar']:
add_chat_history_to_context = gr.Checkbox(label="Include Chat History",
value=kwargs[
'add_chat_history_to_context'])
add_search_to_context = gr.Checkbox(label="Include Web Search",
value=kwargs['add_search_to_context'],
visible=serp_visible)
resources_acc_label = "Resources" if not is_public else "Collections"
langchain_mode_radio_kwargs = dict(
choices=langchain_choices0,
value=kwargs['langchain_mode'],
label="Collections",
show_label=True,
visible=kwargs['langchain_mode'] != 'Disabled',
min_width=100)
if is_public:
langchain_mode = gr.Radio(**langchain_mode_radio_kwargs)
with gr.Accordion(resources_acc_label, open=False, visible=database_visible and not is_public):
if not is_public:
langchain_mode = gr.Radio(**langchain_mode_radio_kwargs)
if kwargs['actions_in_sidebar']:
add_chat_history_to_context = gr.Checkbox(label="Chat History",
value=kwargs['add_chat_history_to_context'])
add_search_to_context = gr.Checkbox(label="Web Search",
value=kwargs['add_search_to_context'],
visible=serp_visible)
document_subset = gr.Radio([x.name for x in DocumentSubset],
label="Subset",
value=DocumentSubset.Relevant.name,
interactive=True,
visible=not is_public,
)
if kwargs['actions_in_sidebar']:
langchain_action = gr.Radio(
allowed_actions,
value=default_action,
label="Action",
visible=len(allowed_actions) > 1)
allowed_agents = [x for x in langchain_agents_list if x in visible_langchain_agents]
if os.getenv('OPENAI_API_KEY') is None and LangChainAgent.JSON.value in allowed_agents:
allowed_agents.remove(LangChainAgent.JSON.value)
if os.getenv('OPENAI_API_KEY') is None and LangChainAgent.PYTHON.value in allowed_agents:
allowed_agents.remove(LangChainAgent.PYTHON.value)
if LangChainAgent.PANDAS.value in allowed_agents:
allowed_agents.remove(LangChainAgent.PANDAS.value)
langchain_agents = gr.Dropdown(
allowed_agents,
value=None,
label="Agents",
multiselect=True,
interactive=True,
visible=not is_public and len(allowed_agents) > 0,
elem_id="langchain_agents",
filterable=False)
can_db_filter = kwargs['langchain_mode'] != 'Disabled' and kwargs['db_type'] in ['chroma',
'chroma_old']
document_choice_kwargs = dict(choices=docs_state0,
label="Document",
value=[DocumentChoice.ALL.value],
interactive=True,
multiselect=True,
visible=can_db_filter,
elem_id="multi-selection",
allow_custom_value=False,
)
if kwargs['document_choice_in_sidebar']:
document_choice = gr.Dropdown(**document_choice_kwargs)
visible_doc_track = upload_visible and kwargs['visible_doc_track'] and not kwargs[
'large_file_count_mode']
row_doc_track = gr.Row(visible=visible_doc_track)
with row_doc_track:
if kwargs['langchain_mode'] in langchain_modes_non_db:
doc_counts_str = "Pure LLM Mode"
else:
doc_counts_str = "Name: %s\nDocs: Unset\nChunks: Unset" % kwargs['langchain_mode']
text_doc_count = gr.Textbox(lines=3, label="Doc Counts", value=doc_counts_str,
visible=visible_doc_track)
text_file_last = gr.Textbox(lines=1, label="Newest Doc", value=None, visible=visible_doc_track)
new_files_last = gr.Textbox(label="New Docs full paths as dict of full file names and content",
value='{}',
visible=False)
text_viewable_doc_count = gr.Textbox(lines=2, label=None, visible=False)
col_tabs = gr.Column(elem_id="col-tabs", scale=10)
with col_tabs, gr.Tabs():
if kwargs['chat_tables']:
chat_tab = gr.Row(visible=True)
else:
chat_tab = gr.TabItem("Chat") \
if kwargs['visible_chat_tab'] else gr.Row(visible=False)
with chat_tab:
if kwargs['langchain_mode'] == 'Disabled':
text_output_nochat = gr.Textbox(lines=5, label=output_label0, show_copy_button=True,
visible=not kwargs['chat'])
else:
# text looks a bit worse, but HTML links work
text_output_nochat = gr.HTML(label=output_label0, visible=not kwargs['chat'])
with gr.Row():
# NOCHAT
instruction_nochat = gr.Textbox(
lines=kwargs['input_lines'],
label=instruction_label_nochat,
placeholder=kwargs['placeholder_instruction'],
visible=not kwargs['chat'],
)
iinput_nochat = gr.Textbox(lines=4, label="Input context for Instruction",
placeholder=kwargs['placeholder_input'],
value=kwargs['iinput'],
visible=not kwargs['chat'])
submit_nochat = gr.Button("Submit", size='sm', visible=not kwargs['chat'])
flag_btn_nochat = gr.Button("Flag", size='sm', visible=not kwargs['chat'])
score_text_nochat = gr.Textbox("Response Score: NA", show_label=False,
visible=not kwargs['chat'])
submit_nochat_api = gr.Button("Submit nochat API", visible=False)
submit_nochat_api_plain = gr.Button("Submit nochat API Plain", visible=False)
inputs_dict_str = gr.Textbox(label='API input for nochat', show_label=False, visible=False)
text_output_nochat_api = gr.Textbox(lines=5, label='API nochat output', visible=False,
show_copy_button=True)
visible_upload = (allow_upload_to_user_data or
allow_upload_to_my_data) and \
kwargs['langchain_mode'] != 'Disabled'
# CHAT
col_chat = gr.Column(visible=kwargs['chat'])
with col_chat:
with gr.Row():
with gr.Column(scale=50):
with gr.Row(elem_id="prompt-form-row"):
label_instruction = 'Ask anything or Ingest'
instruction = gr.Textbox(
lines=kwargs['input_lines'],
label=label_instruction,
info=instruction_label,
# info=None,
elem_id='prompt-form',
container=True,
)
mw0 = 20
mic_button = gr.Button(
elem_id="microphone-button" if kwargs['enable_stt'] else None,
value="đ´",
size="sm",
min_width=mw0,
visible=kwargs['enable_stt'])
attach_button = gr.UploadButton(
elem_id="attach-button" if visible_upload else None,
value=None,
label="Upload",
size="sm",
min_width=mw0,
file_types=['.' + x for x in file_types],
file_count="multiple",
visible=visible_upload)
add_button = gr.Button(
elem_id="add-button" if visible_upload and not kwargs[
'actions_in_sidebar'] else None,
value="Ingest",
size="sm",
min_width=mw0,
visible=visible_upload and not kwargs['actions_in_sidebar'])
# AUDIO
if kwargs['enable_stt']:
def action(btn, instruction1, audio_state1, stt_continue_mode=1):
# print("B0: %s %s" % (audio_state1[0], instruction1), flush=True)
"""Changes button text on click"""
if btn == 'đ´':
audio_state1[3] = 'on'
# print("A: %s %s" % (audio_state1[0], instruction1), flush=True)
if stt_continue_mode == 1:
audio_state1[0] = instruction1
audio_state1[1] = instruction1
audio_state1[2] = None
return 'â', instruction1, audio_state1
else:
audio_state1[3] = 'off'
if stt_continue_mode == 1:
audio_state1[0] = None # indicates done for race case
instruction1 = audio_state1[1]
audio_state1[2] = []
# print("B1: %s %s" % (audio_state1[0], instruction1), flush=True)
return 'đ´', instruction1, audio_state1
# while audio state used, entries are pre_text, instruction source, and audio chunks, condition
audio_state0 = [None, None, None, 'off']
audio_state = gr.State(value=audio_state0)
audio_output = gr.HTML(visible=False)
audio = gr.Audio(**mic_sources_kwargs, streaming=True, visible=False,
# max_length=30 if is_public else None,
elem_id='audio',
# waveform_options=dict(show_controls=True),
)
mic_button_kwargs = dict(fn=functools.partial(action,
stt_continue_mode=kwargs[
'stt_continue_mode']),
inputs=[mic_button, instruction,
audio_state],
outputs=[mic_button, instruction,
audio_state],
api_name='mic' if allow_api else None,
show_progress='hidden')
# JS first, then python, but all in one click instead of using .then() that will delay
mic_button.click(fn=lambda: None, **mic_kwargs, **noqueue_kwargs2) \
.then(**mic_button_kwargs)
audio.stream(fn=kwargs['transcriber_func'],
inputs=[audio_state, audio],
outputs=[audio_state, instruction],
show_progress='hidden')
submit_buttons = gr.Row(equal_height=False, visible=kwargs['visible_submit_buttons'])
with submit_buttons:
mw1 = 50
mw2 = 50
with gr.Column(min_width=mw1):
submit = gr.Button(value='Submit', variant='primary', size='sm',
min_width=mw1, elem_id="submit")
stop_btn = gr.Button(value="Stop", variant='secondary', size='sm',
min_width=mw1, elem_id='stop')
save_chat_btn = gr.Button("Save", size='sm', min_width=mw1)
with gr.Column(min_width=mw2):
retry_btn = gr.Button("Redo", size='sm', min_width=mw2)
undo = gr.Button("Undo", size='sm', min_width=mw2)
clear_chat_btn = gr.Button(value="Clear", size='sm', min_width=mw2)
if kwargs['enable_stt'] and (
kwargs['tts_action_phrases'] or kwargs['tts_stop_phrases']):
def detect_words(action_text1, stop_text1, text):
got_action_word = False
action_words = kwargs['tts_action_phrases']
if action_words:
for action_word in action_words:
if action_word.lower() in text.lower():
text = text[:text.lower().index(action_word.lower())]
print("Got action: %s %s" % (action_text1, text), flush=True)
got_action_word = True
if got_action_word:
action_text1 = action_text1 + '.'
got_stop_word = False
stop_words = kwargs['tts_stop_phrases']
if stop_words:
for stop_word in stop_words:
if stop_word.lower() in text.lower():
text = text[:text.lower().index(stop_word.lower())]
print("Got stop: %s %s" % (stop_text1, text), flush=True)
got_stop_word = True
if got_stop_word:
stop_text1 = stop_text1 + '.'
return action_text1, stop_text1, text
action_text = gr.Textbox(value='', visible=False)
stop_text = gr.Textbox(value='', visible=False)
# avoid if no action word, may take extra time
instruction.change(fn=detect_words,
inputs=[action_text, stop_text, instruction],
outputs=[action_text, stop_text, instruction])
def clear_audio_state():
return audio_state0
action_text.change(fn=clear_audio_state, outputs=audio_state) \
.then(fn=lambda: None, **submit_kwargs)
stop_text.change(fn=clear_audio_state, outputs=audio_state) \
.then(fn=lambda: None, **stop_kwargs)
visible_model_choice = bool(kwargs['model_lock']) and \
len(model_states) > 1 and \
kwargs['visible_visible_models']
with gr.Row(visible=not kwargs['actions_in_sidebar'] or visible_model_choice):
visible_models = gr.Dropdown(kwargs['all_possible_visible_models'],
label="Visible Models",
value=visible_models_state0,
interactive=True,
multiselect=True,
visible=visible_model_choice,
elem_id="multi-selection",
filterable=False,
max_choices=kwargs['max_visible_models'],
)
mw0 = 100
with gr.Column(min_width=mw0):
if not kwargs['actions_in_sidebar']:
langchain_action = gr.Radio(
allowed_actions,
value=default_action,
label='Action',
show_label=visible_model_choice,
visible=True,
min_width=mw0)
text_output, text_output2, text_outputs = make_chatbots(output_label0, output_label0_model2,
**kwargs)
with gr.Row():
with gr.Column(visible=kwargs['score_model']):
score_text = gr.Textbox(res_value,
show_label=False,
visible=True)
score_text2 = gr.Textbox("Response Score2: NA", show_label=False,
visible=False and not kwargs['model_lock'])
doc_selection_tab = gr.TabItem("Document Selection") \
if kwargs['visible_doc_selection_tab'] else gr.Row(visible=False)
with doc_selection_tab:
if kwargs['langchain_mode'] in langchain_modes_non_db:
if langchain_mode == LangChainMode.DISABLED.value:
inactive_collection = "#### Document Q/A Disabled -- Chat only mode"
else:
dlabel1 = 'Choose Resources->Collections and Pick Collection'
inactive_collection = "#### Not Chatting with Any Collection\n%s" % dlabel1
active_collection = gr.Markdown(value=inactive_collection)
else:
dlabel1 = 'Select Subset of Document(s) for Chat with Collection: %s' % kwargs['langchain_mode']
active_collection = gr.Markdown(
value="#### Chatting with Collection: %s" % kwargs['langchain_mode'])
if not kwargs['document_choice_in_sidebar']:
document_choice_kwargs.update(dict(label=dlabel1))
document_choice = gr.Dropdown(**document_choice_kwargs)
with gr.Row():
with gr.Column():
document_source_substrings = gr.Dropdown([], label='Source substrings (post-search filter)',
# info='Post-search filter',
interactive=True,
multiselect=True,
visible=can_db_filter,
allow_custom_value=True,
scale=0,
)
with gr.Column():
document_source_substrings_op = gr.Dropdown(['and', 'or'],
label='Source substrings operation',
interactive=True,
multiselect=False,
visible=can_db_filter,
allow_custom_value=False,
scale=0,
)
with gr.Column():
document_content_substrings = gr.Dropdown([],
label='Content substrings (search-time filter)',
# info="Search-time filter of list of words to pass to where_document={'$contains': word list}",
interactive=True,
multiselect=True,
visible=can_db_filter,
allow_custom_value=True,
scale=0,
)
with gr.Column():
document_content_substrings_op = gr.Dropdown(['and', 'or'],
label='Content substrings operation',
interactive=True,
multiselect=False,
visible=can_db_filter,
allow_custom_value=False,
scale=0,
)
sources_visible = kwargs['langchain_mode'] != 'Disabled' and enable_sources_list
with gr.Row():
with gr.Column(scale=1):
get_sources_btn = gr.Button(value="Update UI with Document(s) from DB", scale=0, size='sm',
visible=sources_visible and kwargs['large_file_count_mode'])
# handle API get sources
get_sources_api_btn = gr.Button(visible=False)
get_sources_api_text = gr.Textbox(visible=False)
get_document_api_btn = gr.Button(visible=False)
get_document_api_text = gr.Textbox(visible=False)
show_sources_btn = gr.Button(value="Show Sources from DB", scale=0, size='sm',
visible=sources_visible and kwargs['large_file_count_mode'])
delete_sources_btn = gr.Button(value="Delete Selected (not by substrings) Sources from DB",
scale=0, size='sm',
visible=sources_visible)
refresh_sources_btn = gr.Button(value="Update DB with new/changed files on disk", scale=0,
size='sm',
visible=sources_visible and allow_upload_to_user_data)
with gr.Column(scale=4):
pass
visible_add_remove_collection = visible_upload
with gr.Row():
with gr.Column(scale=1):
add_placeholder = "e.g. UserData2, shared, user_path2" \
if not is_public else "e.g. MyData2, personal (optional)"
remove_placeholder = "e.g. UserData2" if not is_public else "e.g. MyData2"
new_langchain_mode_text = gr.Textbox(value="", visible=visible_add_remove_collection,
label='Add Collection',
placeholder=add_placeholder,
interactive=True)
remove_langchain_mode_text = gr.Textbox(value="", visible=visible_add_remove_collection,
label='Remove Collection from UI',
placeholder=remove_placeholder,
interactive=True)
purge_langchain_mode_text = gr.Textbox(value="", visible=visible_add_remove_collection,
label='Purge Collection (UI, DB, & source files)',
placeholder=remove_placeholder,
interactive=True)
sync_sources_btn = gr.Button(
value="Synchronize DB and UI [only required if did not login and have shared docs]",
scale=0, size='sm',
visible=sources_visible and allow_upload_to_user_data and not kwargs[
'large_file_count_mode'])
load_langchain = gr.Button(
value="Load Collections State [only required if logged in another user ", scale=0,
size='sm',
visible=False and allow_upload_to_user_data and
kwargs['langchain_mode'] != 'Disabled')
with gr.Column(scale=5):
if kwargs['langchain_mode'] != 'Disabled' and visible_add_remove_collection:
df0 = get_df_langchain_mode_paths(selection_docs_state0, None, dbs1=dbs)
else:
df0 = pd.DataFrame(None)
langchain_mode_path_text = gr.Dataframe(value=df0,
visible=visible_add_remove_collection,
label='LangChain Mode-Path',
show_label=False,
interactive=False)
sources_row = gr.Row(visible=kwargs['langchain_mode'] != 'Disabled' and enable_sources_list,
equal_height=False)
with sources_row:
with gr.Column(scale=1):
file_source = gr.File(interactive=False,
label="Download File w/Sources")
with gr.Column(scale=2):
sources_text = gr.HTML(label='Sources Added')
doc_exception_text = gr.Textbox(value="", label='Document Exceptions',
interactive=False,
visible=kwargs['langchain_mode'] != 'Disabled')
if have_arxiv and have_librosa:
file_types_extra = ' URL YouTube ArXiv TEXT'
elif have_librosa:
file_types_extra = ' URL YouTube TEXT'
elif have_arxiv:
file_types_extra = ' URL ArXiv TEXT'
else:
file_types_extra = ' URL TEXT'
file_types_str = ' '.join(file_types) + file_types_extra
gr.Textbox(value=file_types_str, label='Document Types Supported',
lines=2,
interactive=False,
visible=kwargs['langchain_mode'] != 'Disabled')
doc_view_tab = gr.TabItem("Document Viewer") \
if kwargs['visible_doc_view_tab'] else gr.Row(visible=False)
with doc_view_tab:
with gr.Row(visible=kwargs['langchain_mode'] != 'Disabled'):
with gr.Column(scale=2):
get_viewable_sources_btn = gr.Button(value="Update UI with Document(s) from DB", scale=0,
size='sm',
visible=sources_visible and kwargs[
'large_file_count_mode'])
view_document_choice = gr.Dropdown(viewable_docs_state0,
label="Select Single Document to View",
value=None,
interactive=True,
multiselect=False,
visible=True,
elem_id="single-selection",
)
info_view_raw = "Raw text shown if render of original doc fails"
if is_public:
info_view_raw += " (Up to %s chunks in public portal)" % kwargs['max_raw_chunks']
view_raw_text_checkbox = gr.Checkbox(label="View Database Text", value=False,
info=info_view_raw,
visible=kwargs['db_type'] in ['chroma', 'chroma_old'])
with gr.Column(scale=4):
pass
doc_view = gr.HTML(visible=False)
doc_view2 = gr.Dataframe(visible=False)
doc_view3 = gr.JSON(visible=False)
doc_view4 = gr.Markdown(visible=False)
doc_view5 = gr.HTML(visible=False)
if have_gradio_pdf:
from gradio_pdf import PDF
doc_view6 = PDF(visible=False)
else:
doc_view6 = gr.HTML(visible=False)
doc_view7 = gr.Audio(visible=False)
doc_view8 = gr.Video(visible=False)
chat_tab = gr.TabItem("Chat History") \
if kwargs['visible_chat_history_tab'] else gr.Row(visible=False)
with chat_tab:
with gr.Row():
with gr.Column(scale=1):
remove_chat_btn = gr.Button(value="Remove Selected Saved Chats", visible=True, size='sm')
flag_btn = gr.Button("Flag Current Chat", size='sm')
export_chats_btn = gr.Button(value="Export Chats to Download", size='sm')
with gr.Column(scale=4):
pass
with gr.Row():
chats_file = gr.File(interactive=False, label="Download Exported Chats")
chatsup_output = gr.File(label="Upload Chat File(s)",
file_types=['.json'],
file_count='multiple',
elem_id="warning", elem_classes="feedback")
with gr.Row():
if 'mbart-' in kwargs['model_lower']:
src_lang = gr.Dropdown(list(languages_covered().keys()),
value=kwargs['src_lang'],
label="Input Language")
tgt_lang = gr.Dropdown(list(languages_covered().keys()),
value=kwargs['tgt_lang'],
label="Output Language")
chat_exception_text = gr.Textbox(value="", visible=True, label='Chat Exceptions',
interactive=False)
with gr.Row():
count_chat_tokens_btn = gr.Button(value="Count Chat Tokens",
visible=not is_public and not kwargs['model_lock'],
interactive=not is_public, size='sm')
chat_token_count = gr.Textbox(label="Chat Token Count Result", value=None,
visible=not is_public and not kwargs['model_lock'],
interactive=False)
expert_tab = gr.TabItem("Expert") \
if kwargs['visible_expert_tab'] else gr.Row(visible=False)
with expert_tab:
gr.Markdown("Prompt Control")
with gr.Row():
with gr.Column():
if not kwargs['visible_models_tab']:
# only show here if no models tab
prompt_type = get_prompt_type1(**kwargs)
prompt_type2 = get_prompt_type2(**kwargs)
system_prompt_type = gr.Dropdown(label="System Prompt Type",
info="Choose System Prompt Type",
value=kwargs['system_prompt'],
choices=get_system_prompts(),
filterable=True,
)
system_prompt = gr.Textbox(label='System Prompt',
info="Filled by choice above, or can enter your own custom system prompt. auto means automatic, which will auto-switch to DocQA prompt when using collections.",
value=kwargs['system_prompt'], lines=2)
def show_sys(x):
return x
system_prompt_type.change(fn=show_sys, inputs=system_prompt_type, outputs=system_prompt,
**noqueue_kwargs)
context = gr.Textbox(lines=2, label="System Pre-Context",
info="Directly pre-appended without prompt processing (before Pre-Conversation)",
value=kwargs['context'])
chat_conversation = gr.Textbox(lines=2, label="Pre-Conversation",
info="Pre-append conversation for instruct/chat models as List of tuple of (human, bot)",
value=kwargs['chat_conversation'])
text_context_list = gr.Textbox(lines=2, label="Text Doc Q/A",
info="List of strings, for document Q/A, for bypassing database (i.e. also works in LLM Mode)",
value=kwargs['chat_conversation'],
visible=not is_public, # primarily meant for API
)
iinput = gr.Textbox(lines=2, label="Input for Instruct prompt types",
info="If given for document query, added after query",
value=kwargs['iinput'],
placeholder=kwargs['placeholder_input'],
interactive=not is_public)
with gr.Column():
pre_prompt_query = gr.Textbox(label="Query Pre-Prompt",
info="In prompt template, added before document text chunks",
value=kwargs['pre_prompt_query'] or '')
prompt_query = gr.Textbox(label="Query Prompt",
info="Added after documents",
value=kwargs['prompt_query'] or '')
pre_prompt_summary = gr.Textbox(label="Summary Pre-Prompt",
info="In prompt template, added before documents",
value=kwargs['pre_prompt_summary'] or '')
prompt_summary = gr.Textbox(label="Summary Prompt",
info="In prompt template, added after documents text chunks (if query given, 'Focusing on {query}, ' is pre-appended)",
value=kwargs['prompt_summary'] or '')
hyde_llm_prompt = gr.Textbox(label="HYDE LLM Prompt",
info="When doing HYDE, this is first prompt, and in template the user query comes right after this.",
value=kwargs['hyde_llm_prompt'] or '')
llava_prompt_type = gr.Dropdown(label="LLaVa LLM Prompt Type",
info="Pick pre-defined LLaVa prompt",
value=kwargs['llava_prompt'],
choices=get_llava_prompts(),
filterable=True,
)
llava_prompt = gr.Textbox(label="LLaVa LLM Prompt",
info="LLaVa prompt",
value=kwargs['llava_prompt'],
lines=2)
def show_llava(x):
return x
llava_prompt_type.change(fn=show_llava, inputs=llava_prompt_type, outputs=llava_prompt,
**noqueue_kwargs)
gr.Markdown("Document Control")
with gr.Row(visible=not is_public):
image_audio_loaders = gr.CheckboxGroup(image_audio_loaders_options,
label="Force Image-Audio Reader",
value=image_audio_loaders_options0)
pdf_loaders = gr.CheckboxGroup(pdf_loaders_options,
label="Force PDF Reader",
value=pdf_loaders_options0)
url_loaders = gr.CheckboxGroup(url_loaders_options,
label="Force URL Reader",
info="Set env CRAWL_DEPTH to control depth for Scrape, default is 1 (given page + links on that page)",
value=url_loaders_options0)
jq_schema = gr.Textbox(label="JSON jq_schema", value=jq_schema0)
extract_frames = gr.Slider(value=kwargs['extract_frames'] if not is_public else 5,
step=1,
minimum=0,
maximum=5 if is_public else max(kwargs['extract_frames'], 1000),
label="Number of unique images to extract from videos",
info="If 0, just audio extracted if enabled",
visible=have_fiftyone)
min_top_k_docs, max_top_k_docs, label_top_k_docs = get_minmax_top_k_docs(is_public, True)
top_k_docs = gr.Slider(minimum=min_top_k_docs, maximum=max_top_k_docs, step=1,
value=kwargs['top_k_docs'],
label=label_top_k_docs,
# info="For LangChain",
visible=kwargs['langchain_mode'] != 'Disabled',
interactive=not is_public)
chunk = gr.components.Checkbox(value=kwargs['chunk'],
label="Whether to chunk documents",
info="For LangChain",
visible=kwargs['langchain_mode'] != 'Disabled',
interactive=not is_public)
chunk_size = gr.Number(value=kwargs['chunk_size'],
label="Chunk size for document chunking",
info="For LangChain (ignored if chunk=False)",
minimum=128,
maximum=2048,
visible=kwargs['langchain_mode'] != 'Disabled',
interactive=not is_public,
precision=0)
docs_ordering_type = gr.Radio(
docs_ordering_types,
value=kwargs['docs_ordering_type'],
label="Document Sorting in LLM Context",
visible=True)
docs_token_handling = gr.Radio(
docs_token_handlings,
value=kwargs['docs_token_handling'],
label="Document Handling Mode for filling LLM Context",
visible=True)
docs_joiner = gr.Textbox(label="String to join lists and documents",
value=kwargs['docs_joiner'] or docs_joiner_default)
max_hyde_level = 0 if is_public else 5
hyde_level = gr.Slider(minimum=0, maximum=max_hyde_level, step=1,
value=kwargs['hyde_level'],
label='HYDE level',
info="Whether to use HYDE approach for LLM getting answer to embed (0=disabled, 1=non-doc LLM answer, 2=doc-based LLM answer)",
visible=kwargs['langchain_mode'] != 'Disabled',
interactive=not is_public)
hyde_template = gr.components.Textbox(value='auto',
label="HYDE Embedding Template",
info="HYDE approach for LLM getting answer to embed ('auto' means automatic, else enter template like '{query}'",
visible=True)
hyde_show_only_final = gr.components.Checkbox(value=kwargs['hyde_show_only_final'],
label="Only final HYDE shown",
info="Whether to only show final HYDE result",
visible=True)
doc_json_mode = gr.components.Checkbox(value=kwargs['doc_json_mode'],
label="JSON docs mode",
info="Whether to pass JSON to and get JSON back from LLM",
visible=True)
embed = gr.components.Checkbox(value=True,
label="Embed text",
info="For LangChain, whether to embed text",
visible=False)
gr.Markdown("LLM Control")
with gr.Row():
stream_output = gr.components.Checkbox(label="Stream output",
value=kwargs['stream_output'])
do_sample = gr.Checkbox(label="Sample",
info="Enable sampler (required for use of temperature, top_p, top_k). If temperature=0 is set, this is forced to False.",
value=kwargs['do_sample'])
max_time = gr.Slider(minimum=0, maximum=kwargs['max_max_time'], step=1,
value=min(kwargs['max_max_time'],
kwargs['max_time']), label="Max. time",
info="Max. time to search optimal output.")
temperature = gr.Slider(minimum=0, maximum=2,
value=kwargs['temperature'],
label="Temperature",
info="Lower is deterministic, higher more creative")
top_p = gr.Slider(minimum=1e-3, maximum=1.0 - 1e-3,
value=kwargs['top_p'], label="Top p",
info="Cumulative probability of tokens to sample from")
top_k = gr.Slider(
minimum=1, maximum=100, step=1,
value=kwargs['top_k'], label="Top k",
info='Num. tokens to sample from'
)
penalty_alpha = gr.Slider(
minimum=0.0, maximum=2.0, step=0.01,
value=kwargs['penalty_alpha'], label="penalty_alpha",
info='penalty_alpha>0 and top_k>1 enables contrastive search'
)
# FIXME: https://github.com/h2oai/h2ogpt/issues/106
if os.getenv('TESTINGFAIL'):
max_beams = 8 if not (memory_restriction_level or is_public) else 1
else:
max_beams = 1
num_beams = gr.Slider(minimum=1, maximum=max_beams, step=1,
value=min(max_beams, kwargs['num_beams']), label="Beams",
info="Number of searches for optimal overall probability. "
"Uses more GPU memory/compute",
interactive=False, visible=max_beams > 1)
max_max_new_tokens = get_max_max_new_tokens(model_state0, **kwargs)
max_new_tokens = gr.Slider(
minimum=1, maximum=max_max_new_tokens, step=1,
value=min(max_max_new_tokens, kwargs['max_new_tokens']), label="Max output length",
)
min_new_tokens = gr.Slider(
minimum=0, maximum=max_max_new_tokens, step=1,
value=min(max_max_new_tokens, kwargs['min_new_tokens']), label="Min output length",
)
max_new_tokens2 = gr.Slider(
minimum=1, maximum=max_max_new_tokens, step=1,
value=min(max_max_new_tokens, kwargs['max_new_tokens']), label="Max output length 2",
visible=False and not kwargs['model_lock'],
)
min_new_tokens2 = gr.Slider(
minimum=0, maximum=max_max_new_tokens, step=1,
value=min(max_max_new_tokens, kwargs['min_new_tokens']), label="Min output length 2",
visible=False and not kwargs['model_lock'],
)
min_max_new_tokens = gr.Slider(
minimum=1, maximum=max_max_new_tokens, step=1,
value=min(max_max_new_tokens, kwargs['min_max_new_tokens']),
label="Min. of Max output length",
visible=not is_public,
)
max_input_tokens = gr.Number(
minimum=-1 if not is_public else kwargs['max_input_tokens'],
maximum=128 * 1024 if not is_public else kwargs['max_input_tokens'],
step=1,
value=-1 if not is_public else kwargs['max_input_tokens'],
label="Max input length (treat as if model has more limited context, e.g. for context-filling when top_k_docs=-1)",
visible=not is_public,
)
max_total_input_tokens = gr.Number(
minimum=-1 if not is_public else kwargs['max_total_input_tokens'],
maximum=128 * 1024 if not is_public else kwargs['max_total_input_tokens'],
step=1,
value=-1 if not is_public else kwargs['max_total_input_tokens'],
label="Max input length across all LLM calls when doing summarization/extraction",
visible=not is_public,
)
early_stopping = gr.Checkbox(label="EarlyStopping", info="Stop early in beam search",
value=kwargs['early_stopping'], visible=max_beams > 1)
repetition_penalty = gr.Slider(minimum=0.01, maximum=3.0,
value=kwargs['repetition_penalty'],
label="Repetition Penalty")
num_return_sequences = gr.Slider(minimum=1, maximum=10, step=1,
value=kwargs['num_return_sequences'],
label="Number Returns", info="Must be <= num_beams",
interactive=not is_public, visible=max_beams > 1)
chat = gr.components.Checkbox(label="Chat mode", value=kwargs['chat'],
visible=False, # no longer support nochat in UI
interactive=not is_public,
)
clone_visible = visible = kwargs['enable_tts'] and kwargs['tts_model'].startswith('tts_models/')
if clone_visible:
markdown_label = "Speech Control and Voice Cloning"
else:
markdown_label = "Speech Control"
audio_visible = kwargs['enable_tts'] and kwargs['tts_model']
gr.Markdown(markdown_label, visible=audio_visible)
with gr.Row(visible=audio_visible):
if audio_visible:
speech_human = gr.Audio(value=None,
label="Generated Human Speech",
type="numpy",
streaming=True,
interactive=False,
show_label=True,
autoplay=True,
elem_id='human_audio',
visible=audio_visible)
speech_bot = gr.Audio(value=None,
label="Generated Bot Speech",
type="numpy",
streaming=True,
interactive=False,
show_label=True,
autoplay=True,
elem_id='bot_audio',
visible=audio_visible)
speech_bot2 = gr.Audio(value=None,
label="Generated Bot 2 Speech",
type="numpy",
streaming=True,
interactive=False,
show_label=True,
autoplay=False,
visible=False,
elem_id='bot2_audio')
else:
# Ensure not streaming media, just webconnect, if not doing TTS
speech_human = gr.Textbox(visible=False)
speech_bot = gr.Textbox(visible=False)
speech_bot2 = gr.Textbox(visible=False)
if kwargs['enable_tts'] and kwargs['tts_model'].startswith('tts_models/'):
from src.tts_coqui import get_languages_gr
tts_language = get_languages_gr(visible=True, value=kwargs['tts_language'])
else:
tts_language = gr.Dropdown(visible=False)
def process_audio(file1, t1=0, t2=30):
# use no more than 30 seconds
from pydub import AudioSegment
# in milliseconds
t1 = t1 * 1000
t2 = t2 * 1000
newAudio = AudioSegment.from_wav(file1)[t1:t2]
new_file = file1 + '.new.wav'
newAudio.export(new_file, format="wav")
return new_file
if audio_visible:
model_base = os.getenv('H2OGPT_MODEL_BASE', 'models/')
female_voice = os.path.join(model_base, "female.wav")
ref_voice_clone = gr.Audio(
label="File for Clone (x resets)",
type="filepath",
value=female_voice if os.path.isfile(female_voice) else None,
# max_length=30 if is_public else None,
visible=clone_visible,
)
ref_voice_clone.upload(process_audio, inputs=ref_voice_clone, outputs=ref_voice_clone)
else:
ref_voice_clone = gr.Textbox(visible=False)
if audio_visible:
mic_voice_clone = gr.Audio(
label="Mic for Clone (x resets)",
type="filepath",
**mic_sources_kwargs,
# max_length=30 if is_public else None,
visible=clone_visible,
)
mic_voice_clone.upload(process_audio, inputs=mic_voice_clone, outputs=mic_voice_clone)
else:
mic_voice_clone = gr.Textbox(visible=False)
choose_mic_voice_clone = gr.Checkbox(
label="Use Mic for Cloning",
value=False,
info="If unchecked, uses File",
visible=clone_visible,
)
role_name_to_add = gr.Textbox(value='', info="Name of Speaker to add", label="Speaker Style",
visible=clone_visible)
add_role = gr.Button(value="Clone Voice for new Speech Style", visible=clone_visible)
def add_role_func(name, file, mic, roles1, use_mic):
if use_mic and os.path.isfile(mic):
roles1[name] = mic
elif os.path.isfile(file):
roles1[name] = file
roles1[name] = process_audio(roles1[name])
return gr.Dropdown(choices=list(roles1.keys())), roles1
add_role_event = add_role.click(add_role_func,
inputs=[role_name_to_add, ref_voice_clone, mic_voice_clone,
roles_state,
choose_mic_voice_clone],
outputs=[chatbot_role, roles_state],
api_name='add_role' if allow_api else None,
**noqueue_kwargs2,
)
models_tab = gr.TabItem("Models") if kwargs['visible_models_tab'] else gr.Row(visible=False)
with models_tab:
load_msg = "Load (Download) Model" if not is_public \
else "LOAD-UNLOAD DISABLED FOR HOSTED DEMO"
if kwargs['base_model'] not in ['', None, no_model_str]:
load_msg += ' [WARNING: Avoid --base_model on CLI for memory efficient Load-Unload]'
load_msg2 = load_msg + "2"
variant_load_msg = 'primary' if not is_public else 'secondary'
with gr.Row():
n_gpus_list = [str(x) for x in list(range(-1, n_gpus))]
with gr.Column():
with gr.Row():
with gr.Column(scale=10, visible=not kwargs['model_lock']):
load_model_button = gr.Button(load_msg, variant=variant_load_msg, scale=0,
size='sm', interactive=not is_public)
unload_model_button = gr.Button("UnLoad Model", variant=variant_load_msg, scale=0,
size='sm', interactive=not is_public)
with gr.Row():
with gr.Column():
model_choice = gr.Dropdown(model_options_state.value[0],
label="Choose/Enter Base Model (HF name, TheBloke, file, URL)",
value=kwargs['base_model'] or
model_options_state.value[0],
allow_custom_value=not is_public)
lora_choice = gr.Dropdown(lora_options_state.value[0],
label="Choose/Enter LORA",
value=kwargs['lora_weights'] or
lora_options_state.value[0],
visible=kwargs['show_lora'],
allow_custom_value=not is_public)
server_choice = gr.Dropdown(server_options_state.value[0],
label="Choose/Enter Server",
value=kwargs['inference_server'] or
server_options_state.value[0],
visible=not is_public,
allow_custom_value=not is_public)
if kwargs['visible_models_tab']:
prompt_type = get_prompt_type1(**kwargs)
with gr.Column():
model_used = gr.Textbox(label="Current Model", value=kwargs['base_model'],
interactive=False)
lora_used = gr.Textbox(label="Current LORA", value=kwargs['lora_weights'],
visible=kwargs['show_lora'], interactive=False)
server_used = gr.Textbox(label="Current Server",
value=kwargs['inference_server'],
visible=bool(
kwargs['inference_server']) and not is_public,
interactive=False)
with gr.Column(scale=1, visible=not kwargs['model_lock']):
with gr.Accordion("Precision", open=False, visible=True):
model_load8bit_checkbox = gr.components.Checkbox(
label="Load 8-bit [requires support]",
value=kwargs['load_8bit'], interactive=not is_public)
model_load4bit_checkbox = gr.components.Checkbox(
label="Load 4-bit [requires support]",
value=kwargs['load_4bit'], interactive=not is_public)
model_low_bit_mode = gr.Slider(value=kwargs['low_bit_mode'],
minimum=0, maximum=4, step=1,
label="low_bit_mode",
info="0: no quantization config 1: change compute 2: nf4 3: double quant 4: 2 and 3")
with gr.Accordion("GPU", open=False, visible=n_gpus != 0):
model_use_cpu_checkbox = gr.components.Checkbox(
label="Use CPU even if have GPUs",
value=False,
interactive=not is_public)
model_use_gpu_id_checkbox = gr.components.Checkbox(
label="Choose Devices [If not Checked, use all GPUs]",
value=kwargs['use_gpu_id'],
interactive=not is_public)
llama_multi_gpu_info = "LLaMa.cpp does not support multi-GPU GPU selection, run h2oGPT with env CUDA_VISIBLE_DEVICES set to which GPU to use, else all are used."
model_gpu = gr.Dropdown(n_gpus_list,
label="GPU ID [-1 = all GPUs, if Choose is enabled]",
info=llama_multi_gpu_info,
value=kwargs['gpu_id'],
interactive=not is_public)
with gr.Accordion("Add-ons", open=False, visible=True):
model_attention_sinks = gr.components.Checkbox(
label="Enable Attention Sinks [requires support]",
value=kwargs['attention_sinks'], interactive=not is_public)
model_truncation_generation = gr.components.Checkbox(
label="Truncate generation (disable for attention sinks, enforced if required)",
value=kwargs['truncation_generation'], interactive=not is_public)
model_sink_dict = gr.Textbox(value=str(kwargs['sink_dict'] or {}),
label="sink_dict")
model_load_gptq = gr.Textbox(label="gptq",
info="For TheBloke, use: model",
value=kwargs['load_gptq'],
visible=kwargs['use_autogptq'],
interactive=not is_public)
model_gptq_dict = gr.Textbox(value=str(kwargs['gptq_dict'] or {}),
info="E.g. {'inject_fused_attention':False, 'disable_exllama': True}",
label="gptq_dict",
visible=kwargs['use_autogptq'])
model_load_awq = gr.Textbox(label="awq", value=kwargs['load_awq'],
info="For TheBloke, use: model",
interactive=not is_public)
model_load_exllama_checkbox = gr.components.Checkbox(
label="Load with exllama [requires support]",
value=kwargs['load_exllama'], interactive=not is_public)
model_exllama_dict = gr.Textbox(value=str(kwargs['exllama_dict'] or {}),
label="exllama_dict",
info="E.g. to split across 2 GPUs: {'set_auto_map':20,20}")
hf_label = "HuggingFace" if kwargs['use_autogptq'] else "HuggingFace (inc. GPTQ)"
with gr.Accordion(hf_label, open=False, visible=True):
model_safetensors_checkbox = gr.components.Checkbox(
label="Safetensors [required sometimes, e.g. GPTQ from TheBloke]",
value=kwargs['use_safetensors'], interactive=not is_public)
model_hf_model_dict = gr.Textbox(value=str(kwargs['hf_model_dict'] or {}),
label="hf_model_dict")
model_revision = gr.Textbox(label="revision",
value=kwargs['revision'],
info="Hash on HF to use",
interactive=not is_public)
with gr.Accordion("Current or Custom Model Prompt", open=False, visible=True):
prompt_dict = gr.Textbox(label="Current Prompt (or Custom)",
value=pprint.pformat(kwargs['prompt_dict'] or {},
indent=4),
interactive=not is_public, lines=6)
with gr.Accordion("Current or Custom Context Length", open=False, visible=True):
max_seq_len = gr.Number(value=kwargs['max_seq_len'] or -1,
minimum=-1,
maximum=2 ** 18,
precision=0,
info="If standard LLaMa-2, choose up to 4096 (-1 means choose max of model)",
label="max_seq_len")
max_seq_len_used = gr.Number(value=kwargs['max_seq_len'] or -1,
label="Current Max. Seq. Length",
interactive=False)
rope_scaling = gr.Textbox(value=str(kwargs['rope_scaling'] or {}),
label="rope_scaling",
info="Not required if in config.json. E.g. {'type':'linear', 'factor':4} for HF and {'alpha_value':4} for exllama")
acc_llama = gr.Accordion("LLaMa.cpp & GPT4All", open=False,
visible=kwargs['show_llama'])
with acc_llama:
# with row_llama:
model_path_llama = gr.Textbox(value=kwargs['llamacpp_dict']['model_path_llama'],
lines=4,
label="Choose LLaMa.cpp Model Path/URL (for Base Model: llama)",
visible=kwargs['show_llama'])
n_gpu_layers = gr.Number(value=kwargs['llamacpp_dict']['n_gpu_layers'],
minimum=0, maximum=100,
label="LLaMa.cpp Num. GPU Layers Offloaded",
visible=kwargs['show_llama'])
n_batch = gr.Number(value=kwargs['llamacpp_dict']['n_batch'],
minimum=0, maximum=2048,
label="LLaMa.cpp Batch Size",
visible=kwargs['show_llama'])
n_gqa = gr.Number(value=kwargs['llamacpp_dict']['n_gqa'],
minimum=0, maximum=32,
label="LLaMa.cpp Num. Group Query Attention (8 for 70B LLaMa2)",
visible=kwargs['show_llama'])
llamacpp_dict_more = gr.Textbox(value="{}",
lines=4,
label="Dict for other LLaMa.cpp/GPT4All options",
visible=kwargs['show_llama'])
model_name_gptj = gr.Textbox(value=kwargs['llamacpp_dict']['model_name_gptj'],
label="Choose GPT4All GPTJ Model Path/URL (for Base Model: gptj)",
visible=kwargs['show_gpt4all'])
model_name_gpt4all_llama = gr.Textbox(
value=kwargs['llamacpp_dict']['model_name_gpt4all_llama'],
label="Choose GPT4All LLaMa Model Path/URL (for Base Model: gpt4all_llama)",
visible=kwargs['show_gpt4all'])
col_model2 = gr.Column(visible=False)
with col_model2:
with gr.Row():
with gr.Column(scale=10, visible=not kwargs['model_lock']):
load_model_button2 = gr.Button(load_msg2, variant=variant_load_msg, scale=0,
size='sm', interactive=not is_public)
unload_model_button2 = gr.Button("UnLoad Model2", variant=variant_load_msg, scale=0,
size='sm', interactive=not is_public)
with gr.Row():
with gr.Column():
model_choice2 = gr.Dropdown(model_options_state.value[0],
label="Choose/Enter Model 2 (HF name, TheBloke, file, URL)",
value=no_model_str,
allow_custom_value=not is_public)
lora_choice2 = gr.Dropdown(lora_options_state.value[0],
label="Choose/Enter LORA 2",
value=no_lora_str,
visible=kwargs['show_lora'],
allow_custom_value=not is_public)
server_choice2 = gr.Dropdown(server_options_state.value[0],
label="Choose/Enter Server 2",
value=no_server_str,
visible=not is_public,
allow_custom_value=not is_public)
if kwargs['visible_models_tab']:
prompt_type2 = get_prompt_type2(**kwargs)
with gr.Column():
# no model/lora loaded ever in model2 by default
model_used2 = gr.Textbox(label="Current Model 2", value=no_model_str,
interactive=False)
lora_used2 = gr.Textbox(label="Current LORA (Model 2)", value=no_lora_str,
visible=kwargs['show_lora'], interactive=False)
server_used2 = gr.Textbox(label="Current Server (Model 2)",
value=no_server_str,
interactive=False,
visible=not is_public)
with gr.Column(scale=1, visible=not kwargs['model_lock']):
with gr.Accordion("Precision", open=False, visible=True):
model_load8bit_checkbox2 = gr.components.Checkbox(
label="Load 8-bit (Model 2) [requires support]",
value=kwargs['load_8bit'], interactive=not is_public)
model_load4bit_checkbox2 = gr.components.Checkbox(
label="Load 4-bit (Model 2) [requires support]",
value=kwargs['load_4bit'], interactive=not is_public)
model_low_bit_mode2 = gr.Slider(value=kwargs['low_bit_mode'],
# ok that same as Model 1
minimum=0, maximum=4, step=1,
label="low_bit_mode (Model 2)")
with gr.Accordion("GPU", open=False, visible=n_gpus != 0):
model_use_cpu_checkbox2 = gr.components.Checkbox(
label="Use CPU even if have GPUs (Model 2)",
value=False,
interactive=not is_public)
model_use_gpu_id_checkbox2 = gr.components.Checkbox(
label="Choose Devices (Model 2) [If not Checked, use all GPUs]",
value=kwargs['use_gpu_id'],
interactive=not is_public)
model_gpu2 = gr.Dropdown(n_gpus_list,
label="GPU ID (Model 2) [-1 = all GPUs, if choose is enabled]",
info=llama_multi_gpu_info,
value=kwargs['gpu_id'],
interactive=not is_public)
with gr.Accordion("Add-ons", open=False, visible=True):
model_attention_sinks2 = gr.components.Checkbox(
label="Enable Attention Sinks [requires support] (Model 2)",
value=kwargs['attention_sinks'], interactive=not is_public)
model_truncation_generation2 = gr.components.Checkbox(
label="Truncate generation (disable for attention sinks) (Model 2)",
value=kwargs['truncation_generation'], interactive=not is_public)
model_sink_dict2 = gr.Textbox(value=str(kwargs['sink_dict'] or {}),
label="sink_dict (Model 2)")
model_load_gptq2 = gr.Textbox(label="gptq (Model 2)",
info="For TheBloke models, use: model",
value=kwargs['load_gptq'],
visible=kwargs['use_autogptq'],
interactive=not is_public)
model_gptq_dict2 = gr.Textbox(value=str(kwargs['gptq_dict'] or {}),
info="E.g. {'inject_fused_attention':False, 'disable_exllama': True}",
visible=kwargs['use_autogptq'],
label="gptq_dict (Model 2)")
model_load_awq2 = gr.Textbox(label="awq (Model 2)", value='',
interactive=not is_public)
model_load_exllama_checkbox2 = gr.components.Checkbox(
label="Load with exllama (Model 2) [requires support]",
value=False, interactive=not is_public)
model_exllama_dict2 = gr.Textbox(value=str(kwargs['exllama_dict'] or {}),
label="exllama_dict (Model 2)")
with gr.Accordion(hf_label, open=False, visible=True):
model_safetensors_checkbox2 = gr.components.Checkbox(
label="Safetensors (Model 2) [requires support]",
value=False, interactive=not is_public)
model_hf_model_dict2 = gr.Textbox(value=str(kwargs['hf_model_dict'] or {}),
label="hf_model_dict (Model 2)")
model_revision2 = gr.Textbox(label="revision (Model 2)", value='',
interactive=not is_public)
with gr.Accordion("Current or Custom Model Prompt", open=False, visible=True):
prompt_dict2 = gr.Textbox(label="Current Prompt (or Custom) (Model 2)",
value=pprint.pformat(kwargs['prompt_dict'] or {},
indent=4),
interactive=not is_public, lines=4)
with gr.Accordion("Current or Custom Context Length", open=False, visible=True):
max_seq_len2 = gr.Number(value=kwargs['max_seq_len'] or -1,
minimum=-1,
maximum=2 ** 18,
info="If standard LLaMa-2, choose up to 4096 (-1 means choose max of model)",
label="max_seq_len Model 2")
max_seq_len_used2 = gr.Number(value=-1,
label="mCurrent Max. Seq. Length (Model 2)",
interactive=False)
rope_scaling2 = gr.Textbox(value=str(kwargs['rope_scaling'] or {}),
label="rope_scaling Model 2")
acc_llama2 = gr.Accordion("LLaMa.cpp & GPT4All", open=False,
visible=kwargs['show_llama'])
with acc_llama2:
model_path_llama2 = gr.Textbox(
value=kwargs['llamacpp_dict']['model_path_llama'],
label="Choose LLaMa.cpp Model 2 Path/URL (for Base Model: llama)",
lines=4,
visible=kwargs['show_llama'])
n_gpu_layers2 = gr.Number(value=kwargs['llamacpp_dict']['n_gpu_layers'],
minimum=0, maximum=100,
label="LLaMa.cpp Num. GPU 2 Layers Offloaded",
visible=kwargs['show_llama'])
n_batch2 = gr.Number(value=kwargs['llamacpp_dict']['n_batch'],
minimum=0, maximum=2048,
label="LLaMa.cpp Model 2 Batch Size",
visible=kwargs['show_llama'])
n_gqa2 = gr.Number(value=kwargs['llamacpp_dict']['n_gqa'],
minimum=0, maximum=32,
label="LLaMa.cpp Model 2 Num. Group Query Attention (8 for 70B LLaMa2)",
visible=kwargs['show_llama'])
llamacpp_dict_more2 = gr.Textbox(value="{}",
lines=4,
label="Model 2 Dict for other LLaMa.cpp/GPT4All options",
visible=kwargs['show_llama'])
model_name_gptj2 = gr.Textbox(value=kwargs['llamacpp_dict']['model_name_gptj'],
label="Choose GPT4All GPTJ Model 2 Path/URL (for Base Model: gptj)",
visible=kwargs['show_gpt4all'])
model_name_gpt4all_llama2 = gr.Textbox(
value=kwargs['llamacpp_dict']['model_name_gpt4all_llama'],
label="Choose GPT4All LLaMa Model 2 Path/URL (for Base Model: gpt4all_llama)",
visible=kwargs['show_gpt4all'])
compare_checkbox = gr.components.Checkbox(label="Compare Two Models",
value=kwargs['model_lock'],
visible=not is_public and not kwargs['model_lock'])
with gr.Row(visible=not kwargs['model_lock'] and kwargs['enable_add_models_to_list_ui']):
with gr.Column(scale=50):
new_model = gr.Textbox(label="New Model name/path/URL", interactive=not is_public)
with gr.Column(scale=50):
new_lora = gr.Textbox(label="New LORA name/path/URL", visible=kwargs['show_lora'],
interactive=not is_public)
with gr.Column(scale=50):
new_server = gr.Textbox(label="New Server url:port", interactive=not is_public)
with gr.Row():
add_model_lora_server_button = gr.Button("Add new Model, Lora, Server url:port", scale=0,
variant=variant_load_msg,
size='sm', interactive=not is_public)
system_tab = gr.TabItem("System") \
if kwargs['visible_system_tab'] else gr.Row(visible=False)
with system_tab:
with gr.Row():
with gr.Column(scale=1):
side_bar_text = gr.Textbox('on' if kwargs['visible_side_bar'] else 'off',
visible=False, interactive=False)
doc_count_text = gr.Textbox('on' if kwargs['visible_doc_track'] else 'off',
visible=False, interactive=False)
submit_buttons_text = gr.Textbox('on' if kwargs['visible_submit_buttons'] else 'off',
visible=False, interactive=False)
visible_models_text = gr.Textbox('on' if kwargs['visible_visible_models'] else 'off',
visible=False, interactive=False)
side_bar_btn = gr.Button("Toggle SideBar", variant="secondary", size="sm")
doc_count_btn = gr.Button("Toggle SideBar Document Count/Show Newest", variant="secondary",
size="sm",
visible=langchain_mode != LangChainMode.DISABLED.value)
submit_buttons_btn = gr.Button("Toggle Submit Buttons", variant="secondary", size="sm")
visible_model_btn = gr.Button("Toggle Visible Models", variant="secondary", size="sm")
col_tabs_scale = gr.Slider(minimum=1, maximum=20, value=10, step=1, label='Window Size')
text_outputs_height = gr.Slider(minimum=100, maximum=2000, value=kwargs['height'] or 400,
step=50, label='Chat Height')
pdf_height = gr.Slider(minimum=100, maximum=3000, value=kwargs['pdf_height'] or 800,
step=50, label='PDF Viewer Height',
visible=have_gradio_pdf and langchain_mode != LangChainMode.DISABLED.value)
dark_mode_btn = gr.Button("Dark Mode", variant="secondary", size="sm")
with gr.Column(scale=4):
pass
system_visible0 = not is_public and not admin_pass
admin_row = gr.Row()
with admin_row:
with gr.Column(scale=1):
admin_pass_textbox = gr.Textbox(label="Admin Password",
type='password',
visible=not system_visible0)
with gr.Column(scale=4):
pass
system_row = gr.Row(visible=system_visible0)
with system_row:
with gr.Accordion("Admin", open=False, visible=True):
with gr.Column():
close_btn = gr.Button(value="Shutdown h2oGPT", size='sm',
visible=kwargs['close_button'] and kwargs[
'h2ogpt_pid'] is not None)
with gr.Row():
system_btn = gr.Button(value='Get System Info', size='sm')
system_text = gr.Textbox(label='System Info', interactive=False,
show_copy_button=True)
with gr.Row():
system_input = gr.Textbox(label='System Info Dict Password', interactive=True,
visible=not is_public)
system_btn2 = gr.Button(value='Get System Info Dict', visible=not is_public,
size='sm')
system_text2 = gr.Textbox(label='System Info Dict', interactive=False,
visible=not is_public, show_copy_button=True)
with gr.Row():
system_btn3 = gr.Button(value='Get Hash', visible=not is_public, size='sm')
system_text3 = gr.Textbox(label='Hash', interactive=False,
visible=not is_public, show_copy_button=True)
system_btn4 = gr.Button(value='Get Model Names', visible=not is_public, size='sm')
system_text4 = gr.Textbox(label='Model Names', interactive=False,
visible=not is_public, show_copy_button=True)
with gr.Row():
zip_btn = gr.Button("Zip", size='sm')
zip_text = gr.Textbox(label="Zip file name", interactive=False)
file_output = gr.File(interactive=False, label="Zip file to Download")
with gr.Row():
s3up_btn = gr.Button("S3UP", size='sm')
s3up_text = gr.Textbox(label='S3UP result', interactive=False)
tos_tab = gr.TabItem("Terms of Service") \
if kwargs['visible_tos_tab'] else gr.Row(visible=False)
with tos_tab:
description = ""
description += """
DISCLAIMERS:
etc. added in chat, try to remove some of that to help avoid dup entries when hit new conversation is_same = True # length of conversation has to be same if len(x) != len(y): return False if len(x) != len(y): return False for stepx, stepy in zip(x, y): if len(stepx) != len(stepy): # something off with a conversation return False for stepxx, stepyy in zip(stepx, stepy): if len(stepxx) != len(stepyy): # something off with a conversation return False if len(stepxx) != 2: # something off return False if len(stepyy) != 2: # something off return False questionx = str(stepxx[0]).replace('
', '').replace('
', '') if stepxx[0] is not None else None answerx = str(stepxx[1]).replace('', '').replace('
', '') if stepxx[1] is not None else None questiony = str(stepyy[0]).replace('', '').replace('
', '') if stepyy[0] is not None else None answery = str(stepyy[1]).replace('', '').replace('
', '') if stepyy[1] is not None else None if questionx != questiony or answerx != answery: return False return is_same def save_chat(*args, chat_is_list=False, auth_filename=None, auth_freeze=None, raise_if_none=True): args_list = list(args) db1s = args_list[0] requests_state1 = args_list[1] args_list = args_list[2:] if not chat_is_list: # list of chatbot histories, # can't pass in list with list of chatbot histories and state due to gradio limits chat_list = args_list[:-1] else: assert len(args_list) == 2 chat_list = args_list[0] # if old chat file with single chatbot, get into shape if isinstance(chat_list, list) and len(chat_list) > 0 and isinstance(chat_list[0], list) and len( chat_list[0]) == 2 and isinstance(chat_list[0][0], str) and isinstance(chat_list[0][1], str): chat_list = [chat_list] # remove None histories chat_list_not_none = [x for x in chat_list if x and len(x) > 0 and len(x[0]) == 2 and x[0][1] is not None] chat_list_none = [x for x in chat_list if x not in chat_list_not_none] if len(chat_list_none) > 0 and len(chat_list_not_none) == 0: if raise_if_none: raise ValueError("Invalid chat file") else: chat_state1 = args_list[-1] choices = list(chat_state1.keys()).copy() return chat_state1, gr.update(choices=choices, value=None) # dict with keys of short chat names, values of list of list of chatbot histories chat_state1 = args_list[-1] short_chats = list(chat_state1.keys()) if len(chat_list_not_none) > 0: # make short_chat key from only first history, based upon question that is same anyways chat_first = chat_list_not_none[0] short_chat = get_short_chat(chat_first, short_chats) if short_chat: old_chat_lists = list(chat_state1.values()) already_exists = any([is_chat_same(chat_list, x) for x in old_chat_lists]) if not already_exists: chat_state1[short_chat] = chat_list.copy() # reverse so newest at top choices = list(chat_state1.keys()).copy() choices.reverse() # save saved chats and chatbots to auth file selection_docs_state1 = None langchain_mode2 = None roles_state1 = None model_options_state1 = None lora_options_state1 = None server_options_state1 = None text_output1 = chat_list[0] text_output21 = chat_list[1] text_outputs1 = chat_list[2:] save_auth_func(selection_docs_state1, requests_state1, roles_state1, model_options_state1, lora_options_state1, server_options_state1, chat_state1, langchain_mode2, text_output1, text_output21, text_outputs1, ) return chat_state1, gr.update(choices=choices, value=None) def switch_chat(chat_key, chat_state1, num_model_lock=0): chosen_chat = chat_state1[chat_key] # deal with possible different size of chat list vs. current list ret_chat = [None] * (2 + num_model_lock) for chati in range(0, 2 + num_model_lock): ret_chat[chati % len(ret_chat)] = chosen_chat[chati % len(chosen_chat)] return tuple(ret_chat) def clear_texts(*args): return tuple([[]] * len(args)) def clear_scores(): return gr.Textbox(value=res_value), \ gr.Textbox(value='Response Score: NA'), \ gr.Textbox(value='Response Score: NA') switch_chat_fun = functools.partial(switch_chat, num_model_lock=len(text_outputs)) radio_chats.input(switch_chat_fun, inputs=[radio_chats, chat_state], outputs=[text_output, text_output2] + text_outputs) \ .then(clear_scores, outputs=[score_text, score_text2, score_text_nochat]) def remove_chat(chat_key, chat_state1): if isinstance(chat_key, str): chat_state1.pop(chat_key, None) return gr.update(choices=list(chat_state1.keys()), value=None), chat_state1 remove_chat_event = remove_chat_btn.click(remove_chat, inputs=[radio_chats, chat_state], outputs=[radio_chats, chat_state], **noqueue_kwargs, api_name='remove_chat') def get_chats1(chat_state1): base = 'chats' base = makedirs(base, exist_ok=True, tmp_ok=True, use_base=True) filename = os.path.join(base, 'chats_%s.json' % str(uuid.uuid4())) with open(filename, "wt") as f: f.write(json.dumps(chat_state1, indent=2)) return filename export_chat_event = export_chats_btn.click(get_chats1, inputs=chat_state, outputs=chats_file, **noqueue_kwargs2, api_name='export_chats' if allow_api else None) def add_chats_from_file(db1s, requests_state1, file, chat_state1, radio_chats1, chat_exception_text1, auth_filename=None, auth_freeze=None): if not file: return None, chat_state1, gr.update(choices=list(chat_state1.keys()), value=None), chat_exception_text1 if isinstance(file, str): files = [file] else: files = file if not files: return None, chat_state1, gr.update(choices=list(chat_state1.keys()), value=None), chat_exception_text1 chat_exception_list = [] for file1 in files: try: if hasattr(file1, 'name'): file1 = file1.name with open(file1, "rt") as f: new_chats = json.loads(f.read()) for chat1_k, chat1_v in new_chats.items(): # ignore chat1_k, regenerate and de-dup to avoid loss chat_state1, _ = save_chat(db1s, requests_state1, chat1_v, chat_state1, chat_is_list=True, raise_if_none=True) except BaseException as e: t, v, tb = sys.exc_info() ex = ''.join(traceback.format_exception(t, v, tb)) ex_str = "File %s exception: %s" % (file1, str(e)) print(ex_str, flush=True) chat_exception_list.append(ex_str) chat_exception_text1 = '\n'.join(chat_exception_list) # save chat to auth file selection_docs_state1 = None langchain_mode2 = None roles_state1 = None model_options_state1 = None lora_options_state1 = None server_options_state1 = None text_output1, text_output21, text_outputs1 = None, None, None save_auth_func(selection_docs_state1, requests_state1, roles_state1, model_options_state1, lora_options_state1, server_options_state1, chat_state1, langchain_mode2, text_output1, text_output21, text_outputs1, ) return None, chat_state1, gr.update(choices=list(chat_state1.keys()), value=None), chat_exception_text1 # note for update_user_db_func output is ignored for db chatup_change_eventa = chatsup_output.change(user_state_setup, inputs=[my_db_state, requests_state, langchain_mode], outputs=[my_db_state, requests_state, langchain_mode], show_progress='minimal') add_chats_from_file_func = functools.partial(add_chats_from_file, auth_filename=kwargs['auth_filename'], auth_freeze=kwargs['auth_freeze'], ) chatup_change_event = chatup_change_eventa.then(add_chats_from_file_func, inputs=[my_db_state, requests_state] + [chatsup_output, chat_state, radio_chats, chat_exception_text], outputs=[chatsup_output, chat_state, radio_chats, chat_exception_text], **noqueue_kwargs, api_name='add_to_chats' if allow_api else None) clear_chat_event = clear_chat_btn.click(fn=clear_texts, inputs=[text_output, text_output2] + text_outputs, outputs=[text_output, text_output2] + text_outputs, **noqueue_kwargs, api_name='clear' if allow_api else None) \ .then(deselect_radio_chats, inputs=None, outputs=radio_chats, **noqueue_kwargs) \ .then(clear_scores, outputs=[score_text, score_text2, score_text_nochat]) clear_eventa = save_chat_btn.click(user_state_setup, inputs=[my_db_state, requests_state, langchain_mode], outputs=[my_db_state, requests_state, langchain_mode], show_progress='minimal', **noqueue_kwargs2) save_chat_func = functools.partial(save_chat, auth_filename=kwargs['auth_filename'], auth_freeze=kwargs['auth_freeze'], raise_if_none=False, ) clear_event = clear_eventa.then(save_chat_func, inputs=[my_db_state, requests_state] + [text_output, text_output2] + text_outputs + [chat_state], outputs=[chat_state, radio_chats], api_name='save_chat' if allow_api else None) if kwargs['score_model']: clear_event2 = clear_event.then(clear_scores, outputs=[score_text, score_text2, score_text_nochat]) # NOTE: clear of instruction/iinput for nochat has to come after score, # because score for nochat consumes actual textbox, while chat consumes chat history filled by user() no_chat_args = dict(fn=fun, inputs=[model_state, my_db_state, selection_docs_state, requests_state, roles_state] + inputs_list, outputs=text_output_nochat, queue=queue, ) submit_event_nochat = submit_nochat.click(**no_chat_args, api_name='submit_nochat' if allow_api else None) \ .then(**score_args_nochat, api_name='instruction_bot_score_nochat' if allow_api else None, queue=queue) \ .then(clear_instruct, None, instruction_nochat) \ .then(clear_instruct, None, iinput_nochat) # copy of above with text box submission submit_event_nochat2 = instruction_nochat.submit(**no_chat_args) \ .then(**score_args_nochat, queue=queue) \ .then(clear_instruct, None, instruction_nochat) \ .then(clear_instruct, None, iinput_nochat) submit_event_nochat_api = submit_nochat_api.click(fun_with_dict_str, inputs=[model_state, my_db_state, selection_docs_state, requests_state, roles_state, inputs_dict_str], outputs=text_output_nochat_api, queue=True, # required for generator api_name='submit_nochat_api' if allow_api else None) submit_event_nochat_api_plain = submit_nochat_api_plain.click(fun_with_dict_str_plain, inputs=inputs_dict_str, outputs=text_output_nochat_api, **noqueue_kwargs, api_name='submit_nochat_plain_api' if allow_api else None) def load_model(model_name, lora_weights, server_name, model_state_old, prompt_type_old, load_8bit, load_4bit, low_bit_mode, load_gptq, load_awq, load_exllama, use_safetensors, revision, use_cpu, use_gpu_id, gpu_id, max_seq_len1, rope_scaling1, model_path_llama1, model_name_gptj1, model_name_gpt4all_llama1, n_gpu_layers1, n_batch1, n_gqa1, llamacpp_dict_more1, system_prompt1, exllama_dict, gptq_dict, attention_sinks, sink_dict, truncation_generation, hf_model_dict, model_options_state1, lora_options_state1, server_options_state1, unload=False): if unload: model_name = no_model_str lora_weights = no_lora_str server_name = no_server_str exllama_dict = str_to_dict(exllama_dict) gptq_dict = str_to_dict(gptq_dict) sink_dict = str_to_dict(sink_dict) hf_model_dict = str_to_dict(hf_model_dict) # switch-a-roo on base_model so can pass GGUF/GGML as base model model_name0 = model_name model_name, model_path_llama1, load_gptq, load_awq, n_gqa1 = \ switch_a_roo_llama(model_name, model_path_llama1, load_gptq, load_awq, n_gqa1, kwargs['llamacpp_path']) # after getting results, we always keep all items related to llama.cpp, gptj, gpt4all inside llamacpp_dict llamacpp_dict = str_to_dict(llamacpp_dict_more1) llamacpp_dict.update(dict(model_path_llama=model_path_llama1, model_name_gptj=model_name_gptj1, model_name_gpt4all_llama=model_name_gpt4all_llama1, n_gpu_layers=n_gpu_layers1, n_batch=n_batch1, n_gqa=n_gqa1, )) if model_name == 'llama' and not model_path_llama1: raise ValueError("Must set model_path_llama if model_name==llama") if model_name == 'gptj' and not model_name_gptj: raise ValueError("Must set model_name_gptj if model_name==llama") if model_name == 'gpt4all_llama' and not model_name_gpt4all_llama: raise ValueError("Must set model_name_gpt4all_llama if model_name==llama") # ensure no API calls reach here if is_public: raise RuntimeError("Illegal access for %s" % model_name) # ensure old model removed from GPU memory if kwargs['debug']: print("Pre-switch pre-del GPU memory: %s" % get_torch_allocated(), flush=True) model0 = model_state0['model'] if isinstance(model_state_old['model'], str) and \ model0 is not None and \ hasattr(model0, 'cpu'): # best can do, move model loaded at first to CPU model0.cpu() if model_state_old['model'] is not None and \ not isinstance(model_state_old['model'], str): if hasattr(model_state_old['model'], 'cpu'): try: model_state_old['model'].cpu() except Exception as e: # sometimes hit NotImplementedError: Cannot copy out of meta tensor; no data! print("Unable to put model on CPU: %s" % str(e), flush=True) del model_state_old['model'] model_state_old['model'] = None if model_state_old['tokenizer'] is not None and not isinstance(model_state_old['tokenizer'], str): del model_state_old['tokenizer'] model_state_old['tokenizer'] = None clear_torch_cache(allow_skip=True) if kwargs['debug']: print("Pre-switch post-del GPU memory: %s" % get_torch_allocated(), flush=True) if not model_name: model_name = no_model_str if model_name == no_model_str: # no-op if no model, just free memory # no detranscribe needed for model, never go into evaluate lora_weights = no_lora_str server_name = no_server_str prompt_type_old = '' model_path_llama1 = '' model_name_gptj1 = '' model_name_gpt4all_llama1 = '' load_gptq = '' load_awq = '' return kwargs['model_state_none'].copy(), \ model_name, lora_weights, server_name, \ prompt_type_old, max_seq_len1, \ gr.Slider(maximum=256), \ gr.Slider(maximum=256), \ model_path_llama1, model_name_gptj1, model_name_gpt4all_llama1, \ load_gptq, load_awq, n_gqa1, \ n_batch1, n_gpu_layers1, llamacpp_dict_more1, \ model_options_state1, lora_options_state1, server_options_state1 # don't deepcopy, can contain model itself all_kwargs1 = all_kwargs.copy() all_kwargs1['base_model'] = model_name.strip() all_kwargs1['load_8bit'] = load_8bit all_kwargs1['load_4bit'] = load_4bit all_kwargs1['low_bit_mode'] = low_bit_mode all_kwargs1['load_gptq'] = load_gptq all_kwargs1['load_awq'] = load_awq all_kwargs1['load_exllama'] = load_exllama all_kwargs1['use_safetensors'] = use_safetensors all_kwargs1['revision'] = None if not revision else revision # transcribe, don't pass '' all_kwargs1['use_gpu_id'] = use_gpu_id all_kwargs1['gpu_id'] = int(gpu_id) if gpu_id not in [None, 'None'] else None # detranscribe all_kwargs1['llamacpp_dict'] = llamacpp_dict all_kwargs1['exllama_dict'] = exllama_dict all_kwargs1['gptq_dict'] = gptq_dict all_kwargs1['attention_sinks'] = attention_sinks all_kwargs1['sink_dict'] = sink_dict all_kwargs1['truncation_generation'] = truncation_generation all_kwargs1['hf_model_dict'] = hf_model_dict all_kwargs1['max_seq_len'] = int(max_seq_len1) if max_seq_len1 is not None and max_seq_len1 > 0 else None try: all_kwargs1['rope_scaling'] = str_to_dict(rope_scaling1) # transcribe except: print("Failed to use user input for rope_scaling dict", flush=True) all_kwargs1['rope_scaling'] = {} if use_cpu: all_kwargs1['n_gpus'] = 0 elif use_gpu_id and all_kwargs1['gpu_id']: all_kwargs1['n_gpus'] = 1 else: all_kwargs1['n_gpus'] = n_gpus_global prompt_type1 = model_name_to_prompt_type(model_name, model_name0=model_name0, llamacpp_dict=llamacpp_dict, prompt_type_old=prompt_type_old) # detranscribe if lora_weights == no_lora_str: lora_weights = '' all_kwargs1['lora_weights'] = lora_weights.strip() if server_name == no_server_str: server_name = '' all_kwargs1['inference_server'] = server_name.strip() gradio_model_kwargs = dict(reward_type=False, **get_kwargs(get_model, exclude_names=['reward_type'], **all_kwargs1)) model1, tokenizer1, device1 = get_model_retry(**gradio_model_kwargs) clear_torch_cache() tokenizer_base_model = model_name prompt_dict1, error0 = get_prompt(prompt_type1, '', context='', reduced=False, making_context=False, return_dict=True, system_prompt=system_prompt1) model_state_new = dict(model=model1, tokenizer=tokenizer1, device=device1, base_model=model_name, tokenizer_base_model=tokenizer_base_model, lora_weights=lora_weights, inference_server=server_name, prompt_type=prompt_type1, prompt_dict=prompt_dict1, # FIXME: not typically required, unless want to expose adding h2ogpt endpoint in UI visible_models=None, h2ogpt_key=None, ) max_seq_len1new = get_model_max_length_from_tokenizer(tokenizer1) max_max_new_tokens1 = get_max_max_new_tokens(model_state_new, **kwargs) # FIXME: Ensure stored in login state if model_options_state1 and model_name0 not in model_options_state1[0]: model_options_state1[0].extend([model_name0]) if lora_options_state1 and lora_weights not in lora_options_state1[0]: lora_options_state1[0].extend([lora_weights]) if server_options_state1 and server_name not in server_options_state1[0]: server_options_state1[0].extend([server_name]) if kwargs['debug']: print("Post-switch GPU memory: %s" % get_torch_allocated(), flush=True) return model_state_new, model_name, lora_weights, server_name, \ prompt_type1, max_seq_len1new, \ gr.Slider(maximum=max_max_new_tokens1), \ gr.Slider(maximum=max_max_new_tokens1), \ model_path_llama1, model_name_gptj1, model_name_gpt4all_llama1, \ load_gptq, load_awq, n_gqa1, \ n_batch1, n_gpu_layers1, llamacpp_dict_more1, \ model_options_state1, lora_options_state1, server_options_state1 def get_prompt_str(prompt_type1, prompt_dict1, system_prompt1, which=0): if prompt_type1 in ['', None]: print("Got prompt_type %s: %s" % (which, prompt_type1), flush=True) return str({}) prompt_dict1, prompt_dict_error = get_prompt(prompt_type1, prompt_dict1, context='', reduced=False, making_context=False, return_dict=True, system_prompt=system_prompt1) if prompt_dict_error: return str(prompt_dict_error) else: # return so user can manipulate if want and use as custom return str(prompt_dict1) get_prompt_str_func1 = functools.partial(get_prompt_str, which=1) get_prompt_str_func2 = functools.partial(get_prompt_str, which=2) prompt_type.change(fn=get_prompt_str_func1, inputs=[prompt_type, prompt_dict, system_prompt], outputs=prompt_dict, **noqueue_kwargs) prompt_type2.change(fn=get_prompt_str_func2, inputs=[prompt_type2, prompt_dict2, system_prompt], outputs=prompt_dict2, **noqueue_kwargs) def dropdown_prompt_type_list(x): return gr.Dropdown(value=x) def chatbot_list(x, model_used_in, model_path_llama_in): chat_name = get_chatbot_name(model_used_in, model_path_llama_in) return gr.Textbox(label=chat_name) load_model_inputs = [model_choice, lora_choice, server_choice, model_state, prompt_type, model_load8bit_checkbox, model_load4bit_checkbox, model_low_bit_mode, model_load_gptq, model_load_awq, model_load_exllama_checkbox, model_safetensors_checkbox, model_revision, model_use_cpu_checkbox, model_use_gpu_id_checkbox, model_gpu, max_seq_len, rope_scaling, model_path_llama, model_name_gptj, model_name_gpt4all_llama, n_gpu_layers, n_batch, n_gqa, llamacpp_dict_more, system_prompt, model_exllama_dict, model_gptq_dict, model_attention_sinks, model_sink_dict, model_truncation_generation, model_hf_model_dict, model_options_state, lora_options_state, server_options_state, ] load_model_outputs = [model_state, model_used, lora_used, server_used, # if prompt_type changes, prompt_dict will change via change rule prompt_type, max_seq_len_used, max_new_tokens, min_new_tokens, model_path_llama, model_name_gptj, model_name_gpt4all_llama, model_load_gptq, model_load_awq, n_gqa, n_batch, n_gpu_layers, llamacpp_dict_more, model_options_state, lora_options_state, server_options_state, ] load_model_args = dict(fn=load_model, inputs=load_model_inputs, outputs=load_model_outputs) unload_model_args = dict(fn=functools.partial(load_model, unload=True), inputs=load_model_inputs, outputs=load_model_outputs) prompt_update_args = dict(fn=dropdown_prompt_type_list, inputs=prompt_type, outputs=prompt_type) chatbot_update_args = dict(fn=chatbot_list, inputs=[text_output, model_used, model_path_llama], outputs=text_output) nochat_update_args = dict(fn=chatbot_list, inputs=[text_output_nochat, model_used, model_path_llama], outputs=text_output_nochat) load_model_event = load_model_button.click(**load_model_args, api_name='load_model' if allow_api and not is_public else None) \ .then(**prompt_update_args) \ .then(**chatbot_update_args) \ .then(**nochat_update_args) \ .then(clear_torch_cache) \ .then(**save_auth_kwargs) unload_model_event = unload_model_button.click(**unload_model_args, api_name='unload_model' if allow_api and not is_public else None) \ .then(**prompt_update_args) \ .then(**chatbot_update_args) \ .then(**nochat_update_args) \ .then(clear_torch_cache) load_model_inputs2 = [model_choice2, lora_choice2, server_choice2, model_state2, prompt_type2, model_load8bit_checkbox2, model_load4bit_checkbox2, model_low_bit_mode2, model_load_gptq2, model_load_awq2, model_load_exllama_checkbox2, model_safetensors_checkbox2, model_revision2, model_use_cpu_checkbox2, model_use_gpu_id_checkbox2, model_gpu2, max_seq_len2, rope_scaling2, model_path_llama2, model_name_gptj2, model_name_gpt4all_llama2, n_gpu_layers2, n_batch2, n_gqa2, llamacpp_dict_more2, system_prompt, model_exllama_dict2, model_gptq_dict2, model_attention_sinks2, model_sink_dict2, model_truncation_generation2, model_hf_model_dict2, model_options_state, lora_options_state, server_options_state, ] load_model_outputs2 = [model_state2, model_used2, lora_used2, server_used2, # if prompt_type2 changes, prompt_dict2 will change via change rule prompt_type2, max_seq_len_used2, max_new_tokens2, min_new_tokens2, model_path_llama2, model_name_gptj2, model_name_gpt4all_llama2, model_load_gptq2, model_load_awq2, n_gqa2, n_batch2, n_gpu_layers2, llamacpp_dict_more2, model_options_state, lora_options_state, server_options_state, ] load_model_args2 = dict(fn=load_model, inputs=load_model_inputs2, outputs=load_model_outputs2) unload_model_args2 = dict(fn=functools.partial(load_model, unload=True), inputs=load_model_inputs2, outputs=load_model_outputs2) prompt_update_args2 = dict(fn=dropdown_prompt_type_list, inputs=prompt_type2, outputs=prompt_type2) chatbot_update_args2 = dict(fn=chatbot_list, inputs=[text_output2, model_used2, model_path_llama2], outputs=text_output2) load_model_event2 = load_model_button2.click(**load_model_args2, api_name='load_model2' if allow_api and not is_public else None) \ .then(**prompt_update_args2) \ .then(**chatbot_update_args2) \ .then(clear_torch_cache) \ .then(**save_auth_kwargs) unload_model_event2 = unload_model_button2.click(**unload_model_args2, api_name='unload_model2' if allow_api and not is_public else None) \ .then(**prompt_update_args) \ .then(**chatbot_update_args) \ .then(**nochat_update_args) \ .then(clear_torch_cache) def dropdown_model_lora_server_list(model_list0, model_x, lora_list0, lora_x, server_list0, server_x, model_used1, lora_used1, server_used1, model_used2, lora_used2, server_used2, ): model_new_state = [model_list0[0] + [model_x]] model_new_options = [*model_new_state[0]] if no_model_str in model_new_options: model_new_options.remove(no_model_str) model_new_options = [no_model_str] + sorted(model_new_options) x1 = model_x if model_used1 == no_model_str else model_used1 x2 = model_x if model_used2 == no_model_str else model_used2 ret1 = [gr.Dropdown(value=x1, choices=model_new_options), gr.Dropdown(value=x2, choices=model_new_options), '', model_new_state] lora_new_state = [lora_list0[0] + [lora_x]] lora_new_options = [*lora_new_state[0]] if no_lora_str in lora_new_options: lora_new_options.remove(no_lora_str) lora_new_options = [no_lora_str] + sorted(lora_new_options) # don't switch drop-down to added lora if already have model loaded x1 = lora_x if model_used1 == no_model_str else lora_used1 x2 = lora_x if model_used2 == no_model_str else lora_used2 ret2 = [gr.Dropdown(value=x1, choices=lora_new_options), gr.Dropdown(value=x2, choices=lora_new_options), '', lora_new_state] server_new_state = [server_list0[0] + [server_x]] server_new_options = [*server_new_state[0]] if no_server_str in server_new_options: server_new_options.remove(no_server_str) server_new_options = [no_server_str] + sorted(server_new_options) # don't switch drop-down to added server if already have model loaded x1 = server_x if model_used1 == no_model_str else server_used1 x2 = server_x if model_used2 == no_model_str else server_used2 ret3 = [gr.Dropdown(value=x1, choices=server_new_options), gr.Dropdown(value=x2, choices=server_new_options), '', server_new_state] return tuple(ret1 + ret2 + ret3) add_model_lora_server_event = \ add_model_lora_server_button.click(fn=dropdown_model_lora_server_list, inputs=[model_options_state, new_model] + [lora_options_state, new_lora] + [server_options_state, new_server] + [model_used, lora_used, server_used] + [model_used2, lora_used2, server_used2], outputs=[model_choice, model_choice2, new_model, model_options_state] + [lora_choice, lora_choice2, new_lora, lora_options_state] + [server_choice, server_choice2, new_server, server_options_state], **noqueue_kwargs) go_event = go_btn.click(lambda: gr.update(visible=False), None, go_btn, api_name="go" if allow_api else None, **noqueue_kwargs) \ .then(lambda: gr.update(visible=True), None, normal_block, **noqueue_kwargs) \ .then(**load_model_args, **noqueue_kwargs).then(**prompt_update_args, **noqueue_kwargs) def compare_textbox_fun(x): return gr.Textbox(visible=x) def compare_column_fun(x): return gr.Column(visible=x) def compare_prompt_fun(x): return gr.Dropdown(visible=x) def slider_fun(x): return gr.Slider(visible=x) compare_checkbox.select(compare_textbox_fun, compare_checkbox, text_output2, api_name="compare_checkbox" if allow_api else None) \ .then(compare_column_fun, compare_checkbox, col_model2) \ .then(compare_prompt_fun, compare_checkbox, prompt_type2) \ .then(compare_textbox_fun, compare_checkbox, score_text2) \ .then(slider_fun, compare_checkbox, max_new_tokens2) \ .then(slider_fun, compare_checkbox, min_new_tokens2) # FIXME: add score_res2 in condition, but do better # callback for logging flagged input/output callback.setup(inputs_list + [text_output, text_output2] + text_outputs, "flagged_data_points") flag_btn.click(lambda *args: callback.flag(args), inputs_list + [text_output, text_output2] + text_outputs, None, preprocess=False, api_name='flag' if allow_api else None, **noqueue_kwargs) flag_btn_nochat.click(lambda *args: callback.flag(args), inputs_list + [text_output_nochat], None, preprocess=False, api_name='flag_nochat' if allow_api else None, **noqueue_kwargs) def get_system_info(): if is_public: time.sleep(10) # delay to avoid spam since **noqueue_kwargs return gr.Textbox(value=system_info_print()) system_event = system_btn.click(get_system_info, outputs=system_text, api_name='system_info' if allow_api else None, **noqueue_kwargs) def shutdown_func(h2ogpt_pid): import psutil parent = psutil.Process(h2ogpt_pid) for child in parent.children(recursive=True): child.kill() parent.kill() shutdown_event = close_btn.click(functools.partial(shutdown_func, h2ogpt_pid=kwargs['h2ogpt_pid']), api_name='shutdown' if allow_api and not is_public and kwargs[ 'h2ogpt_pid'] is not None else None, **noqueue_kwargs) def get_system_info_dict(system_input1, **kwargs1): if system_input1 != os.getenv("ADMIN_PASS", ""): return json.dumps({}) exclude_list = ['admin_pass', 'examples'] sys_dict = {k: v for k, v in kwargs1.items() if isinstance(v, (str, int, bool, float)) and k not in exclude_list} try: sys_dict.update(system_info()) except Exception as e: # protection print("Exception: %s" % str(e), flush=True) return json.dumps(sys_dict) system_kwargs = all_kwargs.copy() system_kwargs.update(dict(command=str(' '.join(sys.argv)))) get_system_info_dict_func = functools.partial(get_system_info_dict, **all_kwargs) system_dict_event = system_btn2.click(get_system_info_dict_func, inputs=system_input, outputs=system_text2, api_name='system_info_dict' if allow_api else None, **noqueue_kwargs, # queue to avoid spam ) def get_hash(): return kwargs['git_hash'] system_event = system_btn3.click(get_hash, outputs=system_text3, api_name='system_hash' if allow_api else None, **noqueue_kwargs, ) def get_model_names(): key_list = ['base_model', 'prompt_type', 'prompt_dict'] + list(kwargs['other_model_state_defaults'].keys()) # don't want to expose backend inference server IP etc. # key_list += ['inference_server'] if len(model_states) >= 1: local_model_states = model_states elif model_state0 is not None: local_model_states = [model_state0] else: local_model_states = [] return [{k: x[k] for k in key_list if k in x} for x in local_model_states] models_list_event = system_btn4.click(get_model_names, outputs=system_text4, api_name='model_names' if allow_api else None, **noqueue_kwargs, ) def count_chat_tokens(model_state1, chat1, prompt_type1, prompt_dict1, system_prompt1, chat_conversation1, memory_restriction_level1=0, keep_sources_in_context1=False, ): if model_state1 and not isinstance(model_state1['tokenizer'], str): tokenizer = model_state1['tokenizer'] elif model_state0 and not isinstance(model_state0['tokenizer'], str): tokenizer = model_state0['tokenizer'] else: tokenizer = None if tokenizer is not None: langchain_mode1 = 'LLM' add_chat_history_to_context1 = True # fake user message to mimic bot() chat1 = copy.deepcopy(chat1) chat1 = chat1 + [['user_message1', None]] model_max_length1 = tokenizer.model_max_length context1 = history_to_context(chat1, langchain_mode=langchain_mode1, add_chat_history_to_context=add_chat_history_to_context1, prompt_type=prompt_type1, prompt_dict=prompt_dict1, model_max_length=model_max_length1, memory_restriction_level=memory_restriction_level1, keep_sources_in_context=keep_sources_in_context1, system_prompt=system_prompt1, chat_conversation=chat_conversation1, hyde_level=None, gradio_errors_to_chatbot=kwargs['gradio_errors_to_chatbot']) tokens = tokenizer(context1, return_tensors="pt")['input_ids'] if len(tokens.shape) == 1: return str(tokens.shape[0]) elif len(tokens.shape) == 2: return str(tokens.shape[1]) else: return "N/A" else: return "N/A" count_chat_tokens_func = functools.partial(count_chat_tokens, memory_restriction_level1=memory_restriction_level, keep_sources_in_context1=kwargs['keep_sources_in_context']) count_tokens_event = count_chat_tokens_btn.click(fn=count_chat_tokens_func, inputs=[model_state, text_output, prompt_type, prompt_dict, system_prompt, chat_conversation], outputs=chat_token_count, api_name='count_tokens' if allow_api else None) speak_events = [] if kwargs['enable_tts'] and kwargs['predict_from_text_func'] is not None: if kwargs['tts_model'].startswith('tts_models/'): speak_human_event = speak_human_button.click(kwargs['predict_from_text_func'], inputs=[instruction, chatbot_role, tts_language, roles_state, tts_speed], outputs=speech_human, api_name='speak_human' if allow_api else None, ) speak_events.extend([speak_human_event]) elif kwargs['tts_model'].startswith('microsoft'): speak_human_event = speak_human_button.click(kwargs['predict_from_text_func'], inputs=[instruction, speaker, tts_speed], outputs=speech_human, api_name='speak_human' if allow_api else None, ) speak_events.extend([speak_human_event]) def wrap_pred_func(chatbot_role1, speaker1, tts_language1, roles_state1, tts_speed1, visible_models1, text_output1, text_output21, *args, all_models=[]): # FIXME: Choose first visible text_outputs1 = list(args) text_outputss = [text_output1, text_output21] + text_outputs1 text_outputss = [x[-1][1] for x in text_outputss if len(x) >= 1 and len(x[-1]) == 2 and x[-1][1]] response = text_outputss[0] if text_outputss else '' keep_sources_in_context1 = False langchain_mode1 = None # so always tries hyde_level1 = None # so always tries response = remove_refs(response, keep_sources_in_context1, langchain_mode1, hyde_level1, kwargs['gradio_errors_to_chatbot']) if kwargs['enable_tts'] and kwargs['predict_from_text_func'] is not None and response: if kwargs['tts_model'].startswith('tts_models/') and chatbot_role1 not in [None, 'None']: yield from kwargs['predict_from_text_func'](response, chatbot_role1, tts_language1, roles_state1, tts_speed1) elif kwargs['tts_model'].startswith('microsoft') and speaker1 not in [None, 'None']: yield from kwargs['predict_from_text_func'](response, speaker1, tts_speed1) speak_bot_event = speak_bot_button.click(wrap_pred_func, inputs=[chatbot_role, speaker, tts_language, roles_state, tts_speed, visible_models, text_output, text_output2] + text_outputs, outputs=speech_bot, api_name='speak_bot' if allow_api else None, ) speak_events.extend([speak_bot_event]) def stop_audio_func(): return None, None if kwargs['enable_tts']: stop_speak_button.click(stop_audio_func, outputs=[speech_human, speech_bot], cancels=speak_events, **noqueue_kwargs2) # don't pass text_output, don't want to clear output, just stop it # cancel only stops outer generation, not inner generation or non-generation clear_torch_cache_func_soft = functools.partial(clear_torch_cache, allow_skip=True) stop_event = stop_btn.click(lambda: None, None, None, cancels=submits1 + submits2 + submits3 + submits4 + [submit_event_nochat, submit_event_nochat2] + [eventdb1, eventdb2, eventdb3] + [eventdb7a, eventdb7, eventdb8a, eventdb8, eventdb9a, eventdb9, eventdb12a, eventdb12] + db_events + [eventdbloadla, eventdbloadlb] + [clear_event] + [submit_event_nochat_api, submit_event_nochat] + [load_model_event, load_model_event2] + [count_tokens_event] + speak_events , **noqueue_kwargs, api_name='stop' if allow_api else None) \ .then(clear_torch_cache_func_soft, **noqueue_kwargs) \ .then(stop_audio_func, outputs=[speech_human, speech_bot]) if kwargs['auth'] is not None: auth = authf load_func = user_state_setup load_inputs = [my_db_state, requests_state, login_btn, login_btn] load_outputs = [my_db_state, requests_state, login_btn] else: auth = None load_func, load_inputs, load_outputs = None, None, None app_js = wrap_js_to_lambda( len(load_inputs) if load_inputs else 0, get_dark_js() if kwargs['dark'] else None, get_heap_js(heap_app_id) if is_heap_analytics_enabled else None) load_kwargs = dict(js=app_js) if is_gradio_version4 else dict(_js=app_js) load_event = demo.load(fn=load_func, inputs=load_inputs, outputs=load_outputs, **load_kwargs) if load_func: load_event2 = load_event.then(load_login_func, inputs=login_inputs, outputs=login_outputs) if not kwargs['large_file_count_mode']: load_event3 = load_event2.then(**get_sources_kwargs) load_event4 = load_event3.then(fn=update_dropdown, inputs=docs_state, outputs=document_choice) load_event5 = load_event4.then(**show_sources_kwargs) load_event6 = load_event5.then(**get_viewable_sources_args) load_event7 = load_event6.then(**viewable_kwargs) demo.queue(**queue_kwargs, api_open=kwargs['api_open']) favicon_file = "h2o-logo.svg" favicon_path = kwargs['favicon_path'] or favicon_file if not os.path.isfile(favicon_file): print("favicon_path1=%s not found" % favicon_file, flush=True) alt_path = os.path.dirname(os.path.abspath(__file__)) favicon_path = os.path.join(alt_path, favicon_file) if not os.path.isfile(favicon_path): print("favicon_path2: %s not found in %s" % (favicon_file, alt_path), flush=True) alt_path = os.path.dirname(alt_path) favicon_path = os.path.join(alt_path, favicon_file) if not os.path.isfile(favicon_path): print("favicon_path3: %s not found in %s" % (favicon_file, alt_path), flush=True) favicon_path = None if kwargs['prepare_offline_level'] > 0: from src.prepare_offline import go_prepare_offline go_prepare_offline(**locals()) return scheduler = BackgroundScheduler() if kwargs['clear_torch_cache_level'] in [0, 1]: interval_time = 120 clear_torch_cache_func_periodic = clear_torch_cache_func_soft else: interval_time = 20 clear_torch_cache_func_periodic = clear_torch_cache # don't require ever clear torch cache scheduler.add_job(func=clear_torch_cache_func_periodic, trigger="interval", seconds=interval_time) if is_public and \ kwargs['base_model'] not in non_hf_types: # FIXME: disable for gptj, langchain or gpt4all modify print itself # FIXME: and any multi-threaded/async print will enter model output! scheduler.add_job(func=ping, trigger="interval", seconds=60) if os.getenv('PING_GPU'): scheduler.add_job(func=ping_gpu, trigger="interval", seconds=60 * 10) scheduler.start() # import control if kwargs['langchain_mode'] == 'Disabled' and \ os.environ.get("TEST_LANGCHAIN_IMPORT") and \ kwargs['base_model'] not in non_hf_types: assert 'gpt_langchain' not in sys.modules, "Dev bug, import of langchain when should not have" assert 'langchain' not in sys.modules, "Dev bug, import of langchain when should not have" # set port in case GRADIO_SERVER_PORT was already set in prior main() call, # gradio does not listen if change after import # Keep None if not set so can find an open port above used ports server_port = os.getenv('GRADIO_SERVER_PORT') if server_port is not None: server_port = int(server_port) demo.launch(share=kwargs['share'], server_name=kwargs['server_name'], show_error=True, server_port=server_port, favicon_path=favicon_path, prevent_thread_lock=True, auth=auth, auth_message=auth_message, root_path=kwargs['root_path'], ssl_keyfile=kwargs['ssl_keyfile'], ssl_verify=kwargs['ssl_verify'], ssl_certfile=kwargs['ssl_certfile'], ssl_keyfile_password=kwargs['ssl_keyfile_password'], max_threads=max(128, 4 * kwargs['concurrency_count']) if isinstance(kwargs['concurrency_count'], int) else 128, ) showed_server_name = 'localhost' if kwargs['server_name'] == "0.0.0.0" else kwargs['server_name'] if kwargs['verbose'] or not (kwargs['base_model'] in ['gptj', 'gpt4all_llama']): print("Started Gradio Server and/or GUI: server_name: %s port: %s" % (showed_server_name, server_port), flush=True) if server_port is None: server_port = '7860' if kwargs['open_browser']: # Open URL in a new tab, if a browser window is already open. import webbrowser webbrowser.open_new_tab(demo.local_url) else: print("Use local URL: %s" % demo.local_url, flush=True) if kwargs['openai_server']: from openai_server.server import run url_split = demo.local_url.split(':') if len(url_split) == 3: gradio_prefix = ':'.join(url_split[0:1]).replace('//', '') gradio_host = ':'.join(url_split[1:2]).replace('//', '') gradio_port = ':'.join(url_split[2:]).split('/')[0] else: gradio_prefix = 'http' gradio_host = ':'.join(url_split[0:1]) gradio_port = ':'.join(url_split[1:]).split('/')[0] h2ogpt_key1 = get_one_key(kwargs['h2ogpt_api_keys'], kwargs['enforce_h2ogpt_api_key']) # ensure can reach out openai_host = gradio_host if gradio_host not in ['localhost', '127.0.0.1'] else '0.0.0.0' run(wait=False, host=openai_host, port=kwargs['openai_port'], gradio_prefix=gradio_prefix, gradio_host=gradio_host, gradio_port=gradio_port, h2ogpt_key=h2ogpt_key1) if kwargs['block_gradio_exit']: demo.block_thread() def show_doc(db1s, selection_docs_state1, requests_state1, langchain_mode1, single_document_choice1, view_raw_text_checkbox1, text_context_list1, pdf_height, dbs1=None, load_db_if_exists1=None, db_type1=None, use_openai_embedding1=None, hf_embedding_model1=None, migrate_embedding_model_or_db1=None, auto_migrate_db1=None, verbose1=False, get_userid_auth1=None, max_raw_chunks=1000000, api=False, n_jobs=-1): file = single_document_choice1 document_choice1 = [single_document_choice1] content = None db_documents = [] db_metadatas = [] if db_type1 in ['chroma', 'chroma_old']: assert langchain_mode1 is not None langchain_mode_paths = selection_docs_state1['langchain_mode_paths'] langchain_mode_types = selection_docs_state1['langchain_mode_types'] from src.gpt_langchain import set_userid, get_any_db, get_docs_and_meta set_userid(db1s, requests_state1, get_userid_auth1) top_k_docs = -1 db = get_any_db(db1s, langchain_mode1, langchain_mode_paths, langchain_mode_types, dbs=dbs1, load_db_if_exists=load_db_if_exists1, db_type=db_type1, use_openai_embedding=use_openai_embedding1, hf_embedding_model=hf_embedding_model1, migrate_embedding_model=migrate_embedding_model_or_db1, auto_migrate_db=auto_migrate_db1, for_sources_list=True, verbose=verbose1, n_jobs=n_jobs, ) query_action = False # long chunks like would be used for summarize # the below is as or filter, so will show doc or by chunk, unrestricted from langchain.vectorstores import Chroma if isinstance(db, Chroma): # chroma >= 0.4 if view_raw_text_checkbox1: one_filter = \ [{"source": {"$eq": x}, "chunk_id": {"$gte": 0}} if query_action else {"source": {"$eq": x}, "chunk_id": { "$gte": -1}} for x in document_choice1][0] else: one_filter = \ [{"source": {"$eq": x}, "chunk_id": {"$gte": 0}} if query_action else {"source": {"$eq": x}, "chunk_id": { "$eq": -1}} for x in document_choice1][0] filter_kwargs = dict(filter={"$and": [dict(source=one_filter['source']), dict(chunk_id=one_filter['chunk_id'])]}) else: # migration for chroma < 0.4 one_filter = \ [{"source": {"$eq": x}, "chunk_id": {"$gte": 0}} if query_action else {"source": {"$eq": x}, "chunk_id": { "$eq": -1}} for x in document_choice1][0] if view_raw_text_checkbox1: # like or, full raw all chunk types filter_kwargs = dict(filter=one_filter) else: filter_kwargs = dict(filter={"$and": [dict(source=one_filter['source']), dict(chunk_id=one_filter['chunk_id'])]}) db_documents, db_metadatas = get_docs_and_meta(db, top_k_docs, filter_kwargs=filter_kwargs, text_context_list=text_context_list1) # order documents from langchain.docstore.document import Document docs_with_score = [(Document(page_content=result[0], metadata=result[1] or {}), 0) for result in zip(db_documents, db_metadatas)] doc_chunk_ids = [x.get('chunk_id', -1) for x in db_metadatas] doc_page_ids = [x.get('page', 0) for x in db_metadatas] doc_hashes = [x.get('doc_hash', 'None') for x in db_metadatas] docs_with_score = [x for hx, px, cx, x in sorted(zip(doc_hashes, doc_page_ids, doc_chunk_ids, docs_with_score), key=lambda x: (x[0], x[1], x[2])) # if cx == -1 ] db_metadatas = [x[0].metadata for x in docs_with_score][:max_raw_chunks] db_documents = [x[0].page_content for x in docs_with_score][:max_raw_chunks] # done reordering if view_raw_text_checkbox1: content = [dict_to_html(x) + '\n' + text_to_html(y) for x, y in zip(db_metadatas, db_documents)] else: content = [text_to_html(y) for x, y in zip(db_metadatas, db_documents)] content = '\n'.join(content) content = f"""