Initial

Dockerfile

@@ -2,6 +2,10 @@ FROM python:3.9
 
 WORKDIR /code
 
+COPY ./requirements.txt /code/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
 COPY . .
 
-CMD ["uvicorn", "inference:app", "--host", "0.0.0.0", "--port", "7860"]
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"]

inference.py

@@ -1,33 +0,0 @@
-from huggingface_hub import InferenceClient
-from fastapi import FastAPI
-from pydantic import BaseModel
-
-client = InferenceClient(model="meta-llama/Meta-Llama-3-8B-Instruct")
-
-class Message(BaseModel):
-    role: str
-    content: str
-
-history = [
-    {
-        "role": "assistant",
-        "content": "You are a helpful assistant called Awerbot. You will help visitors of a website with their inquiries, Awersoft is a Software Development company, to contact to Awersoft the visitors should use contact@awersoft.com",
-    }]
-
-
-app = FastAPI()
-
-@app.post('/chat')
-async def chat(messages: list[Message]): 
-
-    for message in messages:
-        history.append({'role':'user', 'content':message.content})
-
-    output = client.chat_completion(
-        model="meta-llama/Meta-Llama-3-8B-Instruct",
-        messages=history,
-        max_tokens=100,
-        stop="assistant"
-        )
-    
-    return output

main.py

@@ -1,53 +1,33 @@
-import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+from huggingface_hub import InferenceClient
 from fastapi import FastAPI
 from pydantic import BaseModel
-torch.random.manual_seed(0)
+
+client = InferenceClient(model="meta-llama/Meta-Llama-3-8B-Instruct")
 
 class Message(BaseModel):
     role: str
     content: str
 
-model = AutoModelForCausalLM.from_pretrained(
-    "microsoft/Phi-3-mini-4k-instruct", 
-    device_map="cpu", 
-    torch_dtype="auto", 
-    trust_remote_code=True, 
-)
-tokenizer = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct")
-
 history = [
-    {"role": "assistant", "content": "You are a helpful digital assistant. Please provide safe, ethical and accurate information to the user."},
-]
-
-pipe = pipeline(
-    "text-generation",
-    model=model,
-    tokenizer=tokenizer,
-)
+    {
+        "role": "assistant",
+        "content": "You are a helpful assistant called Awerbot. You will help visitors of a website with their inquiries, Awersoft is a Software Development company, to contact to Awersoft the visitors should use contact@awersoft.com",
+    }]
 
-generation_args = {
-    "max_new_tokens": 500,
-    "return_full_text": False,
-    "temperature": 0.0,
-    "do_sample": False,
-}
 
+app = FastAPI()
 
-def chat(messages: list[Message]) -> str:
+@app.post('/chat')
+async def chat(messages: list[Message]): 
 
     for message in messages:
         history.append({'role':'user', 'content':message.content})
-        
-    generated_text = pipe(history, **generation_args)
-
-    print('Generated Text', generated_text)
-    history.append({'role':'assistant', 'content':generated_text[0]['generated_text']})
-    return generated_text[0]['generated_text']
-
-app = FastAPI()
-
-@app.post('/chat')
-async def root(messages: list[Message]): 
-   return chat(messages)
 
+    output = client.chat_completion(
+        model="meta-llama/Meta-Llama-3-8B-Instruct",
+        messages=history,
+        max_tokens=100,
+        stop="assistant"
+        )
+    
+    return output

main_.py

@@ -0,0 +1,53 @@
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+from fastapi import FastAPI
+from pydantic import BaseModel
+torch.random.manual_seed(0)
+
+class Message(BaseModel):
+    role: str
+    content: str
+
+model = AutoModelForCausalLM.from_pretrained(
+    "microsoft/Phi-3-mini-4k-instruct", 
+    device_map="cpu", 
+    torch_dtype="auto", 
+    trust_remote_code=True, 
+)
+tokenizer = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct")
+
+history = [
+    {"role": "assistant", "content": "You are a helpful digital assistant. Please provide safe, ethical and accurate information to the user."},
+]
+
+pipe = pipeline(
+    "text-generation",
+    model=model,
+    tokenizer=tokenizer,
+)
+
+generation_args = {
+    "max_new_tokens": 500,
+    "return_full_text": False,
+    "temperature": 0.0,
+    "do_sample": False,
+}
+
+
+def chat(messages: list[Message]) -> str:
+
+    for message in messages:
+        history.append({'role':'user', 'content':message.content})
+        
+    generated_text = pipe(history, **generation_args)
+
+    print('Generated Text', generated_text)
+    history.append({'role':'assistant', 'content':generated_text[0]['generated_text']})
+    return generated_text[0]['generated_text']
+
+app = FastAPI()
+
+@app.post('/chat')
+async def root(messages: list[Message]): 
+   return chat(messages)
+

requirements.txt

@@ -0,0 +1 @@
+huggingface_hub