Spaces:
Sleeping
Sleeping
:gem: [Feature] New examples: chat with openai and post
Browse files- examples/__init__.py +0 -0
- examples/chat_with_openai.py +25 -0
- examples/chat_with_post.py +55 -0
examples/__init__.py
ADDED
File without changes
|
examples/chat_with_openai.py
ADDED
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from openai import OpenAI
|
2 |
+
|
3 |
+
# If runnning this service with proxy, you might need to unset `http(s)_proxy`.
|
4 |
+
base_url = "http://127.0.0.1:23333"
|
5 |
+
api_key = "sk-xxxxx"
|
6 |
+
|
7 |
+
client = OpenAI(base_url=base_url, api_key=api_key)
|
8 |
+
response = client.chat.completions.create(
|
9 |
+
model="mixtral-8x7b",
|
10 |
+
messages=[
|
11 |
+
{
|
12 |
+
"role": "user",
|
13 |
+
"content": "what is your model",
|
14 |
+
}
|
15 |
+
],
|
16 |
+
stream=True,
|
17 |
+
)
|
18 |
+
|
19 |
+
for chunk in response:
|
20 |
+
if chunk.choices[0].delta.content is not None:
|
21 |
+
print(chunk.choices[0].delta.content, end="", flush=True)
|
22 |
+
elif chunk.choices[0].finish_reason == "stop":
|
23 |
+
print()
|
24 |
+
else:
|
25 |
+
pass
|
examples/chat_with_post.py
ADDED
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import ast
|
2 |
+
import httpx
|
3 |
+
import json
|
4 |
+
import re
|
5 |
+
|
6 |
+
# If runnning this service with proxy, you might need to unset `http(s)_proxy`.
|
7 |
+
chat_api = "http://127.0.0.1:23333"
|
8 |
+
api_key = "sk-xxxxx"
|
9 |
+
requests_headers = {}
|
10 |
+
requests_payload = {
|
11 |
+
"model": "mixtral-8x7b",
|
12 |
+
"messages": [
|
13 |
+
{
|
14 |
+
"role": "user",
|
15 |
+
"content": "what is your model",
|
16 |
+
}
|
17 |
+
],
|
18 |
+
"stream": True,
|
19 |
+
}
|
20 |
+
|
21 |
+
with httpx.stream(
|
22 |
+
"POST",
|
23 |
+
chat_api + "/chat/completions",
|
24 |
+
headers=requests_headers,
|
25 |
+
json=requests_payload,
|
26 |
+
timeout=httpx.Timeout(connect=20, read=60, write=20, pool=None),
|
27 |
+
) as response:
|
28 |
+
# https://docs.aiohttp.org/en/stable/streams.html
|
29 |
+
# https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb
|
30 |
+
response_content = ""
|
31 |
+
for line in response.iter_lines():
|
32 |
+
remove_patterns = [r"^\s*data:\s*", r"^\s*\[DONE\]\s*"]
|
33 |
+
for pattern in remove_patterns:
|
34 |
+
line = re.sub(pattern, "", line).strip()
|
35 |
+
|
36 |
+
if line:
|
37 |
+
try:
|
38 |
+
line_data = json.loads(line)
|
39 |
+
except Exception as e:
|
40 |
+
try:
|
41 |
+
line_data = ast.literal_eval(line)
|
42 |
+
except:
|
43 |
+
print(f"Error: {line}")
|
44 |
+
raise e
|
45 |
+
# print(f"line: {line_data}")
|
46 |
+
delta_data = line_data["choices"][0]["delta"]
|
47 |
+
finish_reason = line_data["choices"][0]["finish_reason"]
|
48 |
+
if "role" in delta_data:
|
49 |
+
role = delta_data["role"]
|
50 |
+
if "content" in delta_data:
|
51 |
+
delta_content = delta_data["content"]
|
52 |
+
response_content += delta_content
|
53 |
+
print(delta_content, end="", flush=True)
|
54 |
+
if finish_reason == "stop":
|
55 |
+
print()
|