niuyazhe
feature(nyz): add naive Mistral-7B-Instruct-v0.1 demo
d2f4f1c
raw
history blame
610 Bytes
from functools import lru_cache
from .base import register_llm
from .llm_client import LLMFlaskClient
@lru_cache()
def _get_mistral_7b_instruct_server(host: str, port: int):
from .llm_server import LLMInstance, create_app
core = LLMInstance('Mistral-7B-Instruct-v0.1')
app = create_app(core)
app.run(host=host, port=port)
def ask_mistral_7b_instruct(message: str, **kwargs):
host, port = '0.0.0.0', 8001
_get_mistral_7b_instruct_server(host, port)
client = LLMFlaskClient(host, port)
return client.run(message).strip()
register_llm('mistral-7b', ask_mistral_7b_instruct)