iiced commited on
Commit
5fa76ab
1 Parent(s): a5e94fd

Add application file

Browse files
Files changed (3) hide show
  1. Dockerfile +11 -0
  2. main.py +54 -0
  3. requirements.txt +4 -0
Dockerfile ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM python:3.9
2
+
3
+ WORKDIR /code
4
+
5
+ COPY ./requirements.txt /code/requirements.txt
6
+
7
+ RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
8
+
9
+ COPY . .
10
+
11
+ CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"]
main.py ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import FastAPI
2
+ from pydantic import BaseModel
3
+ from huggingface_hub import InferenceClient
4
+ import uvicorn
5
+
6
+
7
+ app = FastAPI()
8
+
9
+ client = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1")
10
+
11
+ class Item(BaseModel):
12
+ prompt: str
13
+ history: list
14
+ system_prompt: str
15
+ temperature: float = 0.9
16
+ max_new_tokens: int = 256
17
+ top_p: float = 0.95
18
+ repetition_penalty: float = 1.0
19
+
20
+ def format_prompt(message, history):
21
+ prompt = "<s>"
22
+ for user_prompt, bot_response in history:
23
+ prompt += f"[INST] {user_prompt} [/INST]"
24
+ prompt += f" {bot_response}</s> "
25
+ prompt += f"[INST] {message} [/INST]"
26
+ return prompt
27
+
28
+ def generate(item: Item):
29
+ temperature = float(item.temperature)
30
+ if temperature < 1e-2:
31
+ temperature = 1e-2
32
+ top_p = float(item.top_p)
33
+
34
+ generate_kwargs = dict(
35
+ temperature=temperature,
36
+ max_new_tokens=item.max_new_tokens,
37
+ top_p=top_p,
38
+ repetition_penalty=item.repetition_penalty,
39
+ do_sample=True,
40
+ seed=42,
41
+ )
42
+
43
+ formatted_prompt = format_prompt(f"{item.system_prompt}, {item.prompt}", item.history)
44
+ stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
45
+ output = ""
46
+
47
+ for response in stream:
48
+ output += response.token.text
49
+ return output
50
+
51
+ @app.post("/generate/")
52
+ async def generate_text(item: Item):
53
+ return {"response": generate(item)}
54
+
requirements.txt ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ fastapi
2
+ uvicorn
3
+ huggingface_hub
4
+ pydantic