LLM-api / examples /chat_with_openai.py
ka1kuk's picture
Upload 21 files
da27d7f
raw
history blame
651 Bytes
from openai import OpenAI
# If runnning this service with proxy, you might need to unset `http(s)_proxy`.
base_url = "http://127.0.0.1:23333"
api_key = "sk-xxxxx"
client = OpenAI(base_url=base_url, api_key=api_key)
response = client.chat.completions.create(
model="mixtral-8x7b",
messages=[
{
"role": "user",
"content": "what is your model",
}
],
stream=True,
)
for chunk in response:
if chunk.choices[0].delta.content is not None:
print(chunk.choices[0].delta.content, end="", flush=True)
elif chunk.choices[0].finish_reason == "stop":
print()
else:
pass