Hansimov commited on
Commit
f3b1386
1 Parent(s): 5a836c3

:gem: [Feature] New examples: chat with openai and post

Browse files
examples/__init__.py ADDED
File without changes
examples/chat_with_openai.py ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from openai import OpenAI
2
+
3
+ # If runnning this service with proxy, you might need to unset `http(s)_proxy`.
4
+ base_url = "http://127.0.0.1:23333"
5
+ api_key = "sk-xxxxx"
6
+
7
+ client = OpenAI(base_url=base_url, api_key=api_key)
8
+ response = client.chat.completions.create(
9
+ model="mixtral-8x7b",
10
+ messages=[
11
+ {
12
+ "role": "user",
13
+ "content": "what is your model",
14
+ }
15
+ ],
16
+ stream=True,
17
+ )
18
+
19
+ for chunk in response:
20
+ if chunk.choices[0].delta.content is not None:
21
+ print(chunk.choices[0].delta.content, end="", flush=True)
22
+ elif chunk.choices[0].finish_reason == "stop":
23
+ print()
24
+ else:
25
+ pass
examples/chat_with_post.py ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ast
2
+ import httpx
3
+ import json
4
+ import re
5
+
6
+ # If runnning this service with proxy, you might need to unset `http(s)_proxy`.
7
+ chat_api = "http://127.0.0.1:23333"
8
+ api_key = "sk-xxxxx"
9
+ requests_headers = {}
10
+ requests_payload = {
11
+ "model": "mixtral-8x7b",
12
+ "messages": [
13
+ {
14
+ "role": "user",
15
+ "content": "what is your model",
16
+ }
17
+ ],
18
+ "stream": True,
19
+ }
20
+
21
+ with httpx.stream(
22
+ "POST",
23
+ chat_api + "/chat/completions",
24
+ headers=requests_headers,
25
+ json=requests_payload,
26
+ timeout=httpx.Timeout(connect=20, read=60, write=20, pool=None),
27
+ ) as response:
28
+ # https://docs.aiohttp.org/en/stable/streams.html
29
+ # https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb
30
+ response_content = ""
31
+ for line in response.iter_lines():
32
+ remove_patterns = [r"^\s*data:\s*", r"^\s*\[DONE\]\s*"]
33
+ for pattern in remove_patterns:
34
+ line = re.sub(pattern, "", line).strip()
35
+
36
+ if line:
37
+ try:
38
+ line_data = json.loads(line)
39
+ except Exception as e:
40
+ try:
41
+ line_data = ast.literal_eval(line)
42
+ except:
43
+ print(f"Error: {line}")
44
+ raise e
45
+ # print(f"line: {line_data}")
46
+ delta_data = line_data["choices"][0]["delta"]
47
+ finish_reason = line_data["choices"][0]["finish_reason"]
48
+ if "role" in delta_data:
49
+ role = delta_data["role"]
50
+ if "content" in delta_data:
51
+ delta_content = delta_data["content"]
52
+ response_content += delta_content
53
+ print(delta_content, end="", flush=True)
54
+ if finish_reason == "stop":
55
+ print()