Spaces:

iblfe
/

test

Runtime error

App Files Files Community

test / tests /test_inference_servers.py

iblfe

Upload folder using huggingface_hub

b585c7f verified 5 months ago

raw history blame contribute delete

No virus

43.4 kB

	import os
	import subprocess
	import time
	from datetime import datetime
	import pytest

	from src.utils import get_ngpus_vis, makedirs
	from tests.utils import wrap_test_forked, get_inf_port, get_inf_server
	from tests.test_langchain_units import have_openai_key, have_replicate_key
	from src.client_test import run_client_many, test_client_basic_api_lean
	from src.enums import PromptType, LangChainAction


	@pytest.mark.parametrize("base_model",
	['h2oai/h2ogpt-oig-oasst1-512-6_9b',
	'h2oai/h2ogpt-gm-oasst1-en-2048-falcon-7b-v2',
	'llama', 'gptj']
	)
	@pytest.mark.parametrize("force_langchain_evaluate", [False, True])
	@pytest.mark.parametrize("do_langchain", [False, True])
	@pytest.mark.parametrize("enforce_h2ogpt_api_key", [False, True])
	@pytest.mark.parametrize("enforce_h2ogpt_ui_key", [False, True])
	@wrap_test_forked
	def test_gradio_inference_server(base_model, force_langchain_evaluate, do_langchain,
	enforce_h2ogpt_ui_key, enforce_h2ogpt_api_key,
	prompt='Who are you?', stream_output=False, max_new_tokens=256,
	langchain_mode='Disabled', langchain_action=LangChainAction.QUERY.value,
	langchain_agents=[],
	user_path=None,
	langchain_modes=['UserData', 'MyData', 'LLM', 'Disabled'],
	docs_ordering_type='reverse_sort'):
	if enforce_h2ogpt_api_key and base_model != 'h2oai/h2ogpt-oig-oasst1-512-6_9b':
	# no need for so many cases
	return
	if force_langchain_evaluate:
	langchain_mode = 'MyData'
	if do_langchain:
	langchain_mode = 'UserData'
	from tests.utils import make_user_path_test
	user_path = make_user_path_test()
	# from src.gpt_langchain import get_some_dbs_from_hf
	# get_some_dbs_from_hf()

	max_seq_len_client = None
	if base_model in ['h2oai/h2ogpt-oig-oasst1-512-6_9b', 'h2oai/h2ogpt-oasst1-512-12b']:
	prompt_type = PromptType.human_bot.name
	elif base_model in ['h2oai/h2ogpt-gm-oasst1-en-2048-falcon-7b-v2']:
	prompt_type = PromptType.prompt_answer.name
	elif base_model in ['llama']:
	max_seq_len_client = 2048
	prompt_type = PromptType.llama2.name
	elif base_model in ['gptj']:
	max_seq_len_client = 2048
	prompt_type = PromptType.gptj.name
	else:
	raise NotImplementedError(base_model)

	main_kwargs = dict(base_model=base_model, prompt_type=prompt_type, chat=True,
	stream_output=stream_output, gradio=True, num_beams=1, block_gradio_exit=False,
	max_new_tokens=max_new_tokens,
	langchain_mode=langchain_mode, langchain_action=langchain_action,
	langchain_agents=langchain_agents,
	user_path=user_path,
	langchain_modes=langchain_modes,
	docs_ordering_type=docs_ordering_type,
	force_langchain_evaluate=force_langchain_evaluate,
	system_prompt='',
	verbose=True)

	# inference server
	from src.gen import main
	main(**main_kwargs)
	inference_server = get_inf_server()
	inf_port = get_inf_port()

	# server that consumes inference server has different port
	from src.gen import main
	client_port = inf_port + 2 # assume will not use + 2 in testing, + 1 reserved for non-gradio inference servers
	# only case when GRADIO_SERVER_PORT and HOST should appear in tests because using 2 gradio instances
	os.environ['GRADIO_SERVER_PORT'] = str(client_port)
	os.environ['HOST'] = "http://127.0.0.1:%s" % client_port

	h2ogpt_key = 'foodoo#'
	main_kwargs = main_kwargs.copy()
	if enforce_h2ogpt_api_key:
	main_kwargs.update(dict(enforce_h2ogpt_api_key=True, h2ogpt_api_keys=[h2ogpt_key]))
	main_kwargs.update(dict(max_seq_len=max_seq_len_client))
	main(**main_kwargs, inference_server=inference_server)

	# client test to server that only consumes inference server
	from src.client_test import run_client_chat
	res_dict, client = run_client_chat(prompt=prompt, prompt_type=prompt_type, stream_output=stream_output,
	max_new_tokens=max_new_tokens, langchain_mode=langchain_mode,
	langchain_action=langchain_action, langchain_agents=langchain_agents)
	assert res_dict['prompt'] == prompt
	assert res_dict['iinput'] == ''

	# will use HOST from above
	if enforce_h2ogpt_api_key:
	# try without key first
	ret1, ret2, ret3, ret4, ret5, ret6, ret7 = run_client_many(prompt_type=None)
	assert 'Invalid Access Key' in ret1['response']
	assert 'Invalid Access Key' in ret2['response']
	assert 'Invalid Access Key' in ret3['response']
	assert 'Invalid Access Key' in ret4['response']
	assert 'Invalid Access Key' in ret5['response']
	assert 'Invalid Access Key' in ret6['response']
	assert 'Invalid Access Key' in ret7['response']
	ret1, ret2, ret3, ret4, ret5, ret6, ret7 = run_client_many(prompt_type=None, h2ogpt_key='foo')
	assert 'Invalid Access Key' in ret1['response']
	assert 'Invalid Access Key' in ret2['response']
	assert 'Invalid Access Key' in ret3['response']
	assert 'Invalid Access Key' in ret4['response']
	assert 'Invalid Access Key' in ret5['response']
	assert 'Invalid Access Key' in ret6['response']
	assert 'Invalid Access Key' in ret7['response']

	# try normal or with key if enforcing
	ret1, ret2, ret3, ret4, ret5, ret6, ret7 = run_client_many(prompt_type=None,
	h2ogpt_key=h2ogpt_key) # client shouldn't have to specify
	if base_model == 'h2oai/h2ogpt-oig-oasst1-512-6_9b':
	assert 'h2oGPT' in ret1['response']
	assert 'Birds' in ret2['response']
	assert 'Birds' in ret3['response']
	assert 'h2oGPT' in ret4['response']
	assert 'h2oGPT' in ret5['response']
	assert 'h2oGPT' in ret6['response']
	assert 'h2oGPT' in ret7['response']
	elif base_model == 'h2oai/h2ogpt-gm-oasst1-en-2048-falcon-7b-v2':
	assert 'I am a language model trained' in ret1['response'] or \
	'I am a helpful assistant' in ret1['response'] or \
	'I am a chatbot.' in ret1['response'] or \
	'a chat-based assistant that can answer questions' in ret1['response'] or \
	'I am an AI language model' in ret1['response'] or \
	'I am an AI assistant.' in ret1['response']
	assert 'Once upon a time' in ret2['response']
	assert 'Once upon a time' in ret3['response']
	assert 'I am a language model trained' in ret4['response'] or 'I am a helpful assistant' in \
	ret4['response'] or 'I am a chatbot.' in ret4['response'] or \
	'a chat-based assistant that can answer questions' in ret4['response'] or \
	'I am an AI language model' in ret4['response'] or \
	'I am an AI assistant.' in ret4['response']
	assert 'I am a language model trained' in ret5['response'] or 'I am a helpful assistant' in \
	ret5['response'] or 'I am a chatbot.' in ret5['response'] or \
	'a chat-based assistant that can answer questions' in ret5['response'] or \
	'I am an AI language model' in ret5['response'] or \
	'I am an AI assistant.' in ret5['response']
	assert 'I am a language model trained' in ret6['response'] or 'I am a helpful assistant' in \
	ret6['response'] or 'I am a chatbot.' in ret6['response'] or \
	'a chat-based assistant that can answer questions' in ret6['response'] or \
	'I am an AI language model' in ret6['response'] or \
	'I am an AI assistant.' in ret6['response']
	assert 'I am a language model trained' in ret7['response'] or 'I am a helpful assistant' in \
	ret7['response'] or 'I am a chatbot.' in ret7['response'] or \
	'a chat-based assistant that can answer questions' in ret7['response'] or \
	'I am an AI language model' in ret7['response'] or \
	'I am an AI assistant.' in ret7['response']
	elif base_model == 'llama':
	assert 'I am a bot.' in ret1['response'] or 'can I assist you today?' in ret1[
	'response'] or 'How can I assist you?' in ret1['response'] or "I'm LLaMA" in ret1['response']
	assert 'Birds' in ret2['response'] or 'Once upon a time' in ret2['response']
	assert 'Birds' in ret3['response'] or 'Once upon a time' in ret3['response']
	assert 'I am a bot.' in ret4['response'] or 'can I assist you today?' in ret4[
	'response'] or 'How can I assist you?' in ret4['response'] or "I'm LLaMA" in ret4['response']
	assert 'I am a bot.' in ret5['response'] or 'can I assist you today?' in ret5[
	'response'] or 'How can I assist you?' in ret5['response'] or "I'm LLaMA" in ret5['response']
	assert 'I am a bot.' in ret6['response'] or 'can I assist you today?' in ret6[
	'response'] or 'How can I assist you?' in ret6['response'] or "I'm LLaMA" in ret6['response']
	assert 'I am a bot.' in ret7['response'] or 'can I assist you today?' in ret7[
	'response'] or 'How can I assist you?' in ret7['response'] or "I'm LLaMA" in ret7['response']
	elif base_model == 'gptj':
	assert 'I am a bot.' in ret1['response'] or 'can I assist you today?' in ret1[
	'response'] or 'a student at' in ret1['response'] or 'am a person who' in ret1['response'] or 'I am' in \
	ret1['response'] or "I'm a student at" in ret1['response']
	assert 'Birds' in ret2['response'] or 'Once upon a time' in ret2['response']
	assert 'Birds' in ret3['response'] or 'Once upon a time' in ret3['response']
	assert 'I am a bot.' in ret4['response'] or 'can I assist you today?' in ret4[
	'response'] or 'a student at' in ret4['response'] or 'am a person who' in ret4['response'] or 'I am' in \
	ret4['response'] or "I'm a student at" in ret4['response']
	assert 'I am a bot.' in ret5['response'] or 'can I assist you today?' in ret5[
	'response'] or 'a student at' in ret5['response'] or 'am a person who' in ret5['response'] or 'I am' in \
	ret5['response'] or "I'm a student at" in ret5['response']
	assert 'I am a bot.' in ret6['response'] or 'can I assist you today?' in ret6[
	'response'] or 'a student at' in ret6['response'] or 'am a person who' in ret6['response'] or 'I am' in \
	ret6['response'] or "I'm a student at" in ret6['response']
	assert 'I am a bot.' in ret7['response'] or 'can I assist you today?' in ret7[
	'response'] or 'a student at' in ret7['response'] or 'am a person who' in ret7['response'] or 'I am' in \
	ret7['response'] or "I'm a student at" in ret7['response']
	print("DONE", flush=True)


	def run_docker(inf_port, base_model, low_mem_mode=False, do_shared=True):
	datetime_str = str(datetime.now()).replace(" ", "_").replace(":", "_")
	msg = "Starting HF inference %s..." % datetime_str
	print(msg, flush=True)
	home_dir = os.path.expanduser('~')
	makedirs(os.path.join(home_dir, '.cache/huggingface/hub'))
	data_dir = '%s/.cache/huggingface/hub/' % home_dir
	n_gpus = get_ngpus_vis()
	cmd = ["docker"] + ['run',
	'-d',
	'--runtime', 'nvidia',
	] + gpus_cmd() + [
	'--shm-size', '1g',
	'-e', 'HUGGING_FACE_HUB_TOKEN=%s' % os.environ['HUGGING_FACE_HUB_TOKEN'],
	'-p', '%s:80' % inf_port,
	'-v', '%s/.cache/huggingface/hub/:/data' % home_dir,
	'-v', '%s:/data' % data_dir,
	'ghcr.io/huggingface/text-generation-inference:0.9.3',
	'--model-id', base_model,
	'--max-stop-sequences', '6',
	'--sharded', 'false' if n_gpus == 1 or not do_shared else 'true'
	]
	if n_gpus > 1 and do_shared:
	cmd.extend(['--num-shard', '%s' % n_gpus])
	if low_mem_mode:
	cmd.extend(['--max-input-length', '1024',
	'--max-total-tokens', '2048',
	# '--cuda-memory-fraction', '0.3', # for 0.9.4, but too memory hungry
	])
	else:
	cmd.extend(['--max-input-length', '4096',
	'--max-total-tokens', '8192',
	# '--cuda-memory-fraction', '0.8', # for 0.9.4, but too memory hungry
	])

	print(cmd, flush=True)
	docker_hash = subprocess.check_output(cmd).decode().strip()
	import time
	connected = False
	while not connected:
	cmd = 'docker logs %s' % docker_hash
	o = subprocess.check_output(cmd, shell=True, timeout=15)
	connected = 'Connected' in o.decode("utf-8")
	time.sleep(5)
	print("Done starting TGI server: %s" % docker_hash, flush=True)
	return docker_hash


	def gpus_cmd():
	n_gpus = get_ngpus_vis()
	if n_gpus == 1:
	return ['--gpus', 'device=%d' % int(os.getenv('CUDA_VISIBLE_DEVICES', '0'))]
	elif n_gpus > 2:
	# note below if joined loses ' needed
	return ['--gpus', '\"device=%s\"' % os.getenv('CUDA_VISIBLE_DEVICES',
	str(list(range(0, n_gpus))).replace(']', '').replace('[',
	'').replace(
	' ', '')
	)]


	def run_vllm_docker(inf_port, base_model, tokenizer=None):
	if base_model == 'h2oai/h2ogpt-gm-oasst1-en-2048-falcon-7b-v2':
	# 7b has 71 heads, not divisible
	os.environ['CUDA_VISIBLE_DEVICES'] = '0'
	os.system("docker pull gcr.io/vorvan/h2oai/h2ogpt-runtime:0.1.0")
	datetime_str = str(datetime.now()).replace(" ", "_").replace(":", "_")
	msg = "Starting vLLM inference %s..." % datetime_str
	print(msg, flush=True)
	home_dir = os.path.expanduser('~')
	makedirs(os.path.join(home_dir, '.cache/huggingface/hub'))
	n_gpus = get_ngpus_vis()
	cmd = ["docker"] + ['run',
	'-d',
	'--runtime', 'nvidia',
	] + gpus_cmd() + [
	'--shm-size', '10.24g',
	'-e', 'HUGGING_FACE_HUB_TOKEN=%s' % os.environ['HUGGING_FACE_HUB_TOKEN'],
	'-p', '%s:5000' % inf_port,
	'--entrypoint', '/h2ogpt_conda/vllm_env/bin/python3.10',
	'-e', 'NCCL_IGNORE_DISABLED_P2P=1',
	'-v', '/etc/passwd:/etc/passwd:ro',
	'-v', '/etc/group:/etc/group:ro',
	'-u', '%s:%s' % (os.getuid(), os.getgid()),
	'-v', '%s/.cache:/workspace/.cache' % home_dir,
	# '--network', 'host',
	'gcr.io/vorvan/h2oai/h2ogpt-runtime:0.1.0',
	# 'h2ogpt', # use when built locally with vLLM just freshly added
	# 'docker.io/library/h2ogpt', # use when built locally with vLLM just freshly added
	'-m', 'vllm.entrypoints.openai.api_server',
	'--port=5000',
	'--host=0.0.0.0',
	'--model=%s' % base_model,
	'--tensor-parallel-size=%s' % n_gpus,
	'--seed', '1234',
	'--trust-remote-code',
	'--download-dir=/workspace/.cache/huggingface/hub',
	]
	os.environ.pop('CUDA_VISIBLE_DEVICES', None)
	if tokenizer:
	cmd.append('--tokenizer=%s' % tokenizer)

	print(cmd, flush=True)
	print(' '.join(cmd), flush=True)
	docker_hash = subprocess.check_output(cmd).decode().strip()
	import time
	connected = False
	while not connected:
	cmd = 'docker logs %s' % docker_hash
	o = subprocess.check_output(cmd, shell=True, timeout=15)
	connected = 'Uvicorn running on' in o.decode("utf-8")
	# somehow above message doesn't come up
	connected \|= 'GPU blocks' in o.decode("utf-8")
	time.sleep(5)
	print("Done starting vLLM server: %s" % docker_hash, flush=True)
	return docker_hash


	def run_h2ogpt_docker(port, base_model, inference_server=None, max_new_tokens=None):
	os.system("docker pull gcr.io/vorvan/h2oai/h2ogpt-runtime:0.1.0")
	datetime_str = str(datetime.now()).replace(" ", "_").replace(":", "_")
	msg = "Starting h2oGPT %s..." % datetime_str
	print(msg, flush=True)
	home_dir = os.path.expanduser('~')
	makedirs(os.path.join(home_dir, '.cache/huggingface/hub'))
	makedirs(os.path.join(home_dir, 'save'))
	cmd = ["docker"] + ['run',
	'-d',
	'--runtime', 'nvidia',
	] + gpus_cmd() + [
	'--shm-size', '1g',
	'-p', '%s:7860' % port,
	'-v', '%s/.cache:/workspace/.cache/' % home_dir,
	'-v', '%s/save:/workspace/save' % home_dir,
	'-v', '/etc/passwd:/etc/passwd:ro',
	'-v', '/etc/group:/etc/group:ro',
	'-u', '%s:%s' % (os.getuid(), os.getgid()),
	'-e', 'HUGGING_FACE_HUB_TOKEN=%s' % os.environ['HUGGING_FACE_HUB_TOKEN'],
	'--network', 'host',
	'gcr.io/vorvan/h2oai/h2ogpt-runtime:0.1.0',
	# 'h2ogpt', # use when built locally with vLLM just freshly added
	'/workspace/generate.py',
	'--base_model=%s' % base_model,
	'--use_safetensors=True',
	'--save_dir=/workspace/save/',
	'--score_model=None',
	'--max_max_new_tokens=%s' % (max_new_tokens or 2048),
	'--max_new_tokens=%s' % (max_new_tokens or 1024),
	'--num_async=10',
	'--num_beams=1',
	'--top_k_docs=-1',
	'--chat=True',
	'--stream_output=True',
	# '--debug=True',
	]

	if inference_server:
	cmd.extend(['--inference_server=%s' % inference_server])

	print(cmd, flush=True)
	docker_hash = subprocess.check_output(cmd).decode().strip()
	print("Done starting h2oGPT server: %s" % docker_hash, flush=True)
	return docker_hash


	@pytest.mark.parametrize("base_model",
	# FIXME: Can't get 6.9 or 12b (quantized or not) to work on home system, so do falcon only for now
	# ['h2oai/h2ogpt-oig-oasst1-512-6_9b', 'h2oai/h2ogpt-gm-oasst1-en-2048-falcon-7b-v2']
	['h2oai/h2ogpt-gm-oasst1-en-2048-falcon-7b-v2']
	)
	@pytest.mark.parametrize("force_langchain_evaluate", [False, True])
	@pytest.mark.parametrize("do_langchain", [False, True])
	@pytest.mark.parametrize("pass_prompt_type", [False, True, 'custom'])
	@pytest.mark.parametrize("do_model_lock", [False, True])
	@wrap_test_forked
	def test_hf_inference_server(base_model, force_langchain_evaluate, do_langchain, pass_prompt_type, do_model_lock,
	prompt='Who are you?', stream_output=False, max_new_tokens=256,
	langchain_mode='Disabled',
	langchain_action=LangChainAction.QUERY.value,
	langchain_agents=[],
	user_path=None,
	langchain_modes=['UserData', 'MyData', 'LLM', 'Disabled'],
	docs_ordering_type='reverse_sort'):
	# HF inference server
	gradio_port = get_inf_port()
	inf_port = gradio_port + 1
	inference_server = 'http://127.0.0.1:%s' % inf_port
	docker_hash = run_docker(inf_port, base_model, low_mem_mode=True, do_shared=False)

	if force_langchain_evaluate:
	langchain_mode = 'MyData'
	if do_langchain:
	langchain_mode = 'UserData'
	from tests.utils import make_user_path_test
	user_path = make_user_path_test()
	# from src.gpt_langchain import get_some_dbs_from_hf
	# get_some_dbs_from_hf()

	if base_model in ['h2oai/h2ogpt-oig-oasst1-512-6_9b', 'h2oai/h2ogpt-oasst1-512-12b']:
	prompt_type = PromptType.human_bot.name
	else:
	prompt_type = PromptType.prompt_answer.name
	if isinstance(pass_prompt_type, str):
	prompt_type = 'custom'
	prompt_dict = """{'promptA': None, 'promptB': None, 'PreInstruct': None, 'PreInput': None, 'PreResponse': None, 'terminate_response': [], 'chat_sep': '', 'chat_turn_sep': '', 'humanstr': None, 'botstr': None, 'generates_leading_space': False}"""
	else:
	prompt_dict = None
	if not pass_prompt_type:
	prompt_type = None
	if do_model_lock:
	model_lock = [{'inference_server': inference_server, 'base_model': base_model}]
	base_model = None
	inference_server = None
	else:
	model_lock = None
	main_kwargs = dict(base_model=base_model,
	prompt_type=prompt_type,
	prompt_dict=prompt_dict,
	chat=True,
	system_prompt='',
	stream_output=stream_output, gradio=True, num_beams=1, block_gradio_exit=False,
	max_new_tokens=max_new_tokens,
	langchain_mode=langchain_mode,
	langchain_action=langchain_action,
	langchain_agents=langchain_agents,
	user_path=user_path,
	langchain_modes=langchain_modes,
	docs_ordering_type=docs_ordering_type,
	force_langchain_evaluate=force_langchain_evaluate,
	inference_server=inference_server,
	model_lock=model_lock)

	try:
	# server that consumes inference server
	from src.gen import main
	main(**main_kwargs)

	# client test to server that only consumes inference server
	from src.client_test import run_client_chat
	res_dict, client = run_client_chat(prompt=prompt, prompt_type=prompt_type,
	stream_output=stream_output,
	max_new_tokens=max_new_tokens, langchain_mode=langchain_mode,
	langchain_action=langchain_action,
	langchain_agents=langchain_agents,
	prompt_dict=prompt_dict)
	assert res_dict['prompt'] == prompt
	assert res_dict['iinput'] == ''

	# will use HOST from above
	ret1, ret2, ret3, ret4, ret5, ret6, ret7 = run_client_many(prompt_type=None) # client shouldn't have to specify
	# here docker started with falcon before personalization

	if isinstance(pass_prompt_type, str):
	assert 'year old student from the' in ret1['response'] or 'I am a person who is asking you a question' in \
	ret1['response'] or 'year old' in ret1['response']
	assert 'bird' in ret2['response']
	assert 'bird' in ret3['response']
	assert 'year old student from the' in ret4['response'] or 'I am a person who is asking you a question' in \
	ret4['response'] or 'year old' in ret4['response']
	assert 'year old student from the' in ret5['response'] or 'I am a person who is asking you a question' in \
	ret5['response'] or 'year old' in ret5['response']
	assert 'year old student from the' in ret6['response'] or 'I am a person who is asking you a question' in \
	ret6['response'] or 'year old' in ret6['response']
	assert 'year old student from the' in ret7['response'] or 'I am a person who is asking you a question' in \
	ret7['response'] or 'year old' in ret7['response']
	elif base_model == 'h2oai/h2ogpt-oig-oasst1-512-6_9b':
	assert 'h2oGPT' in ret1['response']
	assert 'Birds' in ret2['response']
	assert 'Birds' in ret3['response']
	assert 'h2oGPT' in ret4['response']
	assert 'h2oGPT' in ret5['response']
	assert 'h2oGPT' in ret6['response']
	assert 'h2oGPT' in ret7['response']
	else:
	assert 'I am a language model trained' in ret1['response'] or 'I am a helpful assistant' in \
	ret1['response'] or 'a chat-based assistant' in ret1['response'] or 'am a student' in ret1[
	'response'] or 'I am an AI language model' in ret1['response']
	assert 'Once upon a time' in ret2['response']
	assert 'Once upon a time' in ret3['response']
	assert 'I am a language model trained' in ret4['response'] or 'I am a helpful assistant' in \
	ret4['response'] or 'a chat-based assistant' in ret4['response'] or 'am a student' in ret4[
	'response'] or 'I am an AI language model' in ret4['response']
	assert 'I am a language model trained' in ret5['response'] or 'I am a helpful assistant' in \
	ret5['response'] or 'a chat-based assistant' in ret5['response'] or 'am a student' in ret5[
	'response'] or 'I am an AI language model' in ret5['response']
	assert 'I am a language model trained' in ret6['response'] or 'I am a helpful assistant' in \
	ret6['response'] or 'a chat-based assistant' in ret6['response'] or 'am a student' in ret6[
	'response'] or 'I am an AI language model' in ret6['response']
	assert 'I am a language model trained' in ret7['response'] or 'I am a helpful assistant' in \
	ret7['response'] or 'a chat-based assistant' in ret7['response'] or 'am a student' in ret7[
	'response'] or 'I am an AI language model' in ret7['response']
	print("DONE", flush=True)
	finally:
	os.system("docker stop %s" % docker_hash)


	chat_conversation1 = [['Who are you?',
	'I am an AI language model created by OpenAI, designed to assist with various tasks such as answering questions, generating text, and providing information.']]


	@pytest.mark.skipif(not have_openai_key, reason="requires OpenAI key to run")
	@pytest.mark.parametrize("system_prompt", ['You are a baby cat who likes to talk to people.', ''])
	@pytest.mark.parametrize("chat_conversation", [chat_conversation1, []])
	@pytest.mark.parametrize("force_langchain_evaluate", [False, True])
	@pytest.mark.parametrize("inference_server", ['openai_chat', 'openai_azure_chat'])
	@wrap_test_forked
	def test_openai_inference_server(inference_server, force_langchain_evaluate, chat_conversation,
	system_prompt,
	prompt='Who are you?', stream_output=False, max_new_tokens=256,
	base_model='gpt-3.5-turbo',
	langchain_mode='Disabled',
	langchain_action=LangChainAction.QUERY.value,
	langchain_agents=[],
	user_path=None,
	langchain_modes=['UserData', 'MyData', 'LLM', 'Disabled'],
	docs_ordering_type='reverse_sort'):
	if force_langchain_evaluate:
	langchain_mode = 'MyData'
	if inference_server == 'openai_azure_chat':
	# need at least deployment name added:
	deployment_name = 'h2ogpt'
	inference_server += ':%s:%s' % (deployment_name, 'h2ogpt.openai.azure.com/')
	if 'azure' in inference_server:
	assert 'OPENAI_AZURE_KEY' in os.environ, "Missing 'OPENAI_AZURE_KEY'"
	os.environ['OPENAI_API_KEY'] = os.environ['OPENAI_AZURE_KEY']

	main_kwargs = dict(base_model=base_model, chat=True,
	stream_output=stream_output, gradio=True, num_beams=1, block_gradio_exit=False,
	max_new_tokens=max_new_tokens,
	langchain_mode=langchain_mode,
	langchain_action=langchain_action,
	langchain_agents=langchain_agents,
	user_path=user_path,
	langchain_modes=langchain_modes,
	system_prompt='auto',
	docs_ordering_type=docs_ordering_type,
	# chat_conversation=chat_conversation # not enough if API passes [], API will override
	)

	# server that consumes inference server
	from src.gen import main
	main(**main_kwargs, inference_server=inference_server)

	if chat_conversation:
	prompt = 'What did I ask?'

	# client test to server that only consumes inference server
	from src.client_test import run_client_chat
	res_dict, client = run_client_chat(prompt=prompt, prompt_type='openai_chat', stream_output=stream_output,
	max_new_tokens=max_new_tokens, langchain_mode=langchain_mode,
	langchain_action=langchain_action, langchain_agents=langchain_agents,
	chat_conversation=chat_conversation,
	system_prompt=system_prompt)
	assert res_dict['prompt'] == prompt
	assert res_dict['iinput'] == ''

	if chat_conversation and system_prompt:
	# TODO: don't check yet, system_prompt ignored if response from LLM is as if no system prompt
	return

	if chat_conversation or system_prompt:
	ret6, _ = test_client_basic_api_lean(prompt=prompt, prompt_type=None,
	chat_conversation=chat_conversation,
	system_prompt=system_prompt)
	if system_prompt:
	assert 'baby cat' in res_dict['response'] and 'meow' in res_dict['response'].lower()
	assert 'baby cat' in ret6['response'] and 'meow' in ret6['response'].lower()
	else:
	options_response = ['You asked "Who are you?"', """You asked, \"Who are you?\""""]
	assert res_dict['response'] in options_response
	assert ret6['response'] in options_response

	return

	if system_prompt:
	# don't test rest, too many cases
	return

	# will use HOST from above
	ret1, ret2, ret3, ret4, ret5, ret6, ret7 = run_client_many(prompt_type=None) # client shouldn't have to specify
	assert 'I am an AI language model' in ret1['response'] or 'I am a helpful assistant designed' in ret1[
	'response'] or 'I am an AI assistant designed to help answer questions and provide information' in ret1[
	'response']
	assert 'Once upon a time, in a far-off land,' in ret2['response'] or 'Once upon a time' in ret2['response']
	assert 'Once upon a time, in a far-off land,' in ret3['response'] or 'Once upon a time' in ret3['response']
	assert 'I am an AI language model' in ret4['response'] or 'I am a helpful assistant designed' in ret4[
	'response'] or 'I am an AI assistant designed to help answer questions and provide information' in ret4[
	'response']
	assert 'I am an AI language model' in ret5['response'] or 'I am a helpful assistant designed' in ret5[
	'response'] or 'I am an AI assistant designed to help answer questions and provide information' in ret5[
	'response']
	assert 'I am an AI language model' in ret6['response'] or 'I am a helpful assistant designed' in ret6[
	'response'] or 'I am an AI assistant designed to help answer questions and provide information' in ret6[
	'response']
	assert 'I am an AI language model' in ret7['response'] or 'I am a helpful assistant designed' in ret7[
	'response'] or 'I am an AI assistant designed to help answer questions and provide information' in ret7[
	'response']
	print("DONE", flush=True)


	@pytest.mark.parametrize("base_model",
	['h2oai/h2ogpt-gm-oasst1-en-2048-falcon-7b-v2', 'meta-llama/Llama-2-7b-chat-hf']
	)
	@wrap_test_forked
	def test_gradio_tgi_docker(base_model):
	# HF inference server
	gradio_port = get_inf_port()
	inf_port = gradio_port + 1
	inference_server = 'http://127.0.0.1:%s' % inf_port
	docker_hash1 = run_docker(inf_port, base_model, low_mem_mode=True, do_shared=False)
	os.system('docker logs %s \| tail -10' % docker_hash1)

	# h2oGPT server
	docker_hash2 = run_h2ogpt_docker(gradio_port, base_model, inference_server=inference_server)
	time.sleep(30) # assumes image already downloaded, else need more time
	os.system('docker logs %s \| tail -10' % docker_hash2)

	# test this version for now, until docker updated
	version = 1

	try:
	# client test to server that only consumes inference server
	prompt = 'Who are you?'
	print("Starting client tests with prompt: %s using %s" % (prompt, get_inf_server()))
	from src.client_test import run_client_chat
	res_dict, client = run_client_chat(prompt=prompt,
	stream_output=True,
	max_new_tokens=256,
	langchain_mode='Disabled',
	langchain_action=LangChainAction.QUERY.value,
	langchain_agents=[],
	version=version)
	assert res_dict['prompt'] == prompt
	assert res_dict['iinput'] == ''

	# will use HOST from above
	# client shouldn't have to specify
	ret1, ret2, ret3, ret4, ret5, ret6, ret7 = run_client_many(prompt_type=None, version=version)
	if 'llama' in base_model.lower():
	who = "I'm LLaMA, an AI assistant developed by Meta AI"
	assert who in ret1['response']
	assert who in ret1['response']
	assert 'Once upon a time' in ret2['response']
	assert 'Once upon a time' in ret3['response']
	assert who in ret4['response']
	assert who in ret5['response']
	assert who in ret6['response']
	assert who in ret7['response']
	else:
	who = 'I am an AI language model'
	assert who in ret1['response']
	assert 'Once upon a time' in ret2['response']
	assert 'Once upon a time' in ret3['response']
	assert who in ret4['response']
	assert who in ret5['response']
	assert who in ret6['response']
	assert who in ret7['response']
	print("DONE", flush=True)
	finally:
	os.system("docker stop %s" % docker_hash1)
	os.system("docker stop %s" % docker_hash2)


	@pytest.mark.parametrize("base_model",
	[
	'h2oai/h2ogpt-gm-oasst1-en-2048-falcon-7b-v2',
	'h2oai/h2ogpt-4096-llama2-7b-chat'] # avoid meta to avoid hassle of key
	)
	@wrap_test_forked
	def test_gradio_vllm_docker(base_model):
	# HF inference server
	gradio_port = get_inf_port()
	inf_port = gradio_port + 1
	inference_server = 'vllm:127.0.0.1:%s' % inf_port
	if 'llama' in base_model:
	tokenizer = 'hf-internal-testing/llama-tokenizer'
	else:
	tokenizer = None

	docker_hash1 = run_vllm_docker(inf_port, base_model, tokenizer)
	os.system('docker logs %s \| tail -10' % docker_hash1)

	# h2oGPT server
	docker_hash2 = run_h2ogpt_docker(gradio_port, base_model, inference_server=inference_server)
	time.sleep(30) # assumes image already downloaded, else need more time
	os.system('docker logs %s \| tail -10' % docker_hash2)

	# test this version for now, until docker updated
	version = 1

	try:
	# client test to server that only consumes inference server
	prompt = 'Who are you?'
	print("Starting client tests with prompt: %s using %s" % (prompt, get_inf_server()))
	from src.client_test import run_client_chat
	res_dict, client = run_client_chat(prompt=prompt,
	stream_output=True,
	max_new_tokens=256,
	langchain_mode='Disabled',
	langchain_action=LangChainAction.QUERY.value,
	langchain_agents=[],
	version=version)
	assert res_dict['prompt'] == prompt
	assert res_dict['iinput'] == ''

	# will use HOST from above
	# client shouldn't have to specify
	ret1, ret2, ret3, ret4, ret5, ret6, ret7 = run_client_many(prompt_type=None, version=version)
	if 'llama' in base_model.lower():
	who = "I'm LLaMA, an AI assistant developed by Meta AI"
	assert who in ret1['response']
	assert who in ret1['response']
	assert 'Once upon a time' in ret2['response']
	assert 'Once upon a time' in ret3['response']
	assert who in ret4['response']
	assert who in ret5['response']
	assert who in ret6['response']
	assert who in ret7['response']
	else:
	who = 'I am an AI language model'
	assert who in ret1['response']
	assert 'Once upon a time' in ret2['response']
	assert 'Once upon a time' in ret3['response']
	assert who in ret4['response']
	assert who in ret5['response']
	assert who in ret6['response']
	assert who in ret7['response']
	print("DONE", flush=True)
	finally:
	os.system("docker stop %s" % docker_hash1)
	os.system("docker stop %s" % docker_hash2)


	@pytest.mark.skipif(not have_replicate_key, reason="requires Replicate key to run")
	@pytest.mark.parametrize("system_prompt", ['You are a baby cat who likes to talk to people.', ''])
	@pytest.mark.parametrize("chat_conversation", [chat_conversation1, []])
	@pytest.mark.parametrize("force_langchain_evaluate", [False, True])
	@wrap_test_forked
	def test_replicate_inference_server(force_langchain_evaluate,
	chat_conversation,
	system_prompt,
	prompt='Who are you?', stream_output=False,
	max_new_tokens=128, # limit cost
	base_model='h2oai/h2ogpt-4096-llama2-7b-chat',
	langchain_mode='Disabled',
	langchain_action=LangChainAction.QUERY.value,
	langchain_agents=[],
	user_path=None,
	langchain_modes=['UserData', 'MyData', 'LLM', 'Disabled'],
	docs_ordering_type='reverse_sort'):
	if force_langchain_evaluate:
	langchain_mode = 'MyData'

	main_kwargs = dict(base_model=base_model, chat=True,
	stream_output=stream_output, gradio=True, num_beams=1, block_gradio_exit=False,
	max_new_tokens=max_new_tokens,
	langchain_mode=langchain_mode,
	langchain_action=langchain_action,
	langchain_agents=langchain_agents,
	user_path=user_path,
	langchain_modes=langchain_modes,
	docs_ordering_type=docs_ordering_type)

	# server that consumes inference server
	from src.gen import main
	# https://replicate.com/lucataco/llama-2-7b-chat
	#model_string = "lucataco/llama-2-7b-chat:6ab580ab4eef2c2b440f2441ec0fc0ace5470edaf2cbea50b8550aec0b3fbd38"
	model_string = "meta/llama-2-7b-chat:8e6975e5ed6174911a6ff3d60540dfd4844201974602551e10e9e87ab143d81e"
	main(**main_kwargs, inference_server='replicate:%s' % model_string)

	if chat_conversation:
	prompt = 'What did I ask?'

	# client test to server that only consumes inference server
	from src.client_test import run_client_chat
	res_dict, client = run_client_chat(prompt=prompt, prompt_type='llama2', stream_output=stream_output,
	max_new_tokens=max_new_tokens, langchain_mode=langchain_mode,
	langchain_action=langchain_action, langchain_agents=langchain_agents,
	chat_conversation=chat_conversation,
	system_prompt=system_prompt)
	assert res_dict['prompt'] == prompt
	assert res_dict['iinput'] == ''

	if chat_conversation and system_prompt:
	# TODO: don't check yet, system_prompt ignored if response from LLM is as if no system prompt
	return

	if chat_conversation or system_prompt:
	ret6, _ = test_client_basic_api_lean(prompt=prompt, prompt_type=None,
	chat_conversation=chat_conversation,
	system_prompt=system_prompt)
	if system_prompt:
	assert 'baby cat' in res_dict['response'] and 'meow' in res_dict['response'].lower()
	assert 'baby cat' in ret6['response'] and 'meow' in ret6['response'].lower()
	else:
	options_response = ['You asked "Who are you?"',
	"""You asked, \"Who are you?\"""",
	"""You asked: \"Who are you?\"""",
	]
	assert res_dict['response'] in options_response
	assert ret6['response'] in options_response

	return

	if system_prompt:
	# don't test rest, too many cases
	return

	# will use HOST from above
	ret1, ret2, ret3, ret4, ret5, ret6, ret7 = run_client_many(prompt_type=None) # client shouldn't have to specify
	who = 'an AI assistant'
	who2 = 'just an AI'
	assert who in ret1['response'] or who2 in ret1['response']
	assert 'Once upon a time, in a far-off land,' in ret2['response'] or 'Once upon a time' in ret2['response']
	assert 'Once upon a time, in a far-off land,' in ret3['response'] or 'Once upon a time' in ret3['response']
	assert who in ret4['response'] or 'I am a helpful assistant designed' in ret4['response'] or who2 in ret4['response']
	assert who in ret5['response'] or 'I am a helpful assistant designed' in ret5['response'] or who2 in ret5['response']
	assert who in ret6['response'] or 'I am a helpful assistant designed' in ret6['response'] or who2 in ret6['response']
	assert who in ret7['response'] or 'I am a helpful assistant designed' in ret7['response'] or who2 in ret7['response']
	print("DONE", flush=True)