Initial

Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3.9
+
+WORKDIR /code
+
+COPY . .
+
+CMD ["uvicorn", "app.inference:app", "--host", "0.0.0.0", "--port", "7860"]

inference.py

@@ -0,0 +1,33 @@
+from huggingface_hub import InferenceClient
+from fastapi import FastAPI
+from pydantic import BaseModel
+
+client = InferenceClient(model="meta-llama/Meta-Llama-3-8B-Instruct")
+
+class Message(BaseModel):
+    role: str
+    content: str
+
+history = [
+    {
+        "role": "assistant",
+        "content": "You are a helpful assistant called Awerbot. You will help visitors of a website with their inquiries, Awersoft is a Software Development company, to contact to Awersoft the visitors should use contact@awersoft.com",
+    }]
+
+
+app = FastAPI()
+
+@app.post('/chat')
+async def chat(messages: list[Message]): 
+
+    for message in messages:
+        history.append({'role':'user', 'content':message.content})
+
+    output = client.chat_completion(
+        model="meta-llama/Meta-Llama-3-8B-Instruct",
+        messages=history,
+        max_tokens=100,
+        stop="assistant"
+        )
+    
+    return output

main.py

@@ -0,0 +1,53 @@
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+from fastapi import FastAPI
+from pydantic import BaseModel
+torch.random.manual_seed(0)
+
+class Message(BaseModel):
+    role: str
+    content: str
+
+model = AutoModelForCausalLM.from_pretrained(
+    "microsoft/Phi-3-mini-4k-instruct", 
+    device_map="cpu", 
+    torch_dtype="auto", 
+    trust_remote_code=True, 
+)
+tokenizer = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct")
+
+history = [
+    {"role": "assistant", "content": "You are a helpful digital assistant. Please provide safe, ethical and accurate information to the user."},
+]
+
+pipe = pipeline(
+    "text-generation",
+    model=model,
+    tokenizer=tokenizer,
+)
+
+generation_args = {
+    "max_new_tokens": 500,
+    "return_full_text": False,
+    "temperature": 0.0,
+    "do_sample": False,
+}
+
+
+def chat(messages: list[Message]) -> str:
+
+    for message in messages:
+        history.append({'role':'user', 'content':message.content})
+        
+    generated_text = pipe(history, **generation_args)
+
+    print('Generated Text', generated_text)
+    history.append({'role':'assistant', 'content':generated_text[0]['generated_text']})
+    return generated_text[0]['generated_text']
+
+app = FastAPI()
+
+@app.post('/chat')
+async def root(messages: list[Message]): 
+   return chat(messages)
+

test.py

@@ -0,0 +1,53 @@
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+from fastapi import FastAPI
+from pydantic import BaseModel
+torch.random.manual_seed(0)
+
+class Message(BaseModel):
+    role: str
+    content: str
+
+model = AutoModelForCausalLM.from_pretrained(
+    "microsoft/Phi-3-mini-4k-instruct", 
+    device_map="cpu", 
+    torch_dtype="auto", 
+    trust_remote_code=True, 
+)
+tokenizer = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct")
+
+history = [
+    {"role": "assistant", "content": "You are a helpful digital assistant. Please provide safe, ethical and accurate information to the user."},
+]
+
+pipe = pipeline(
+    "text-generation",
+    model=model,
+    tokenizer=tokenizer,
+)
+
+generation_args = {
+    "max_new_tokens": 500,
+    "return_full_text": False,
+    "temperature": 0.0,
+    "do_sample": False,
+}
+
+
+def chat(messages: list[Message]) -> str:
+
+    for message in messages:
+        history.append({'role':'user', 'content':message.content})
+        
+    generated_text = pipe(history, **generation_args)
+
+    print('Generated Text', generated_text)
+    history.append({'role':'assistant', 'content':generated_text[0]['generated_text']})
+    return generated_text[0]['generated_text']
+
+app = FastAPI()
+
+@app.post('/chat')
+async def root(messages: list[Message]): 
+   return chat(messages)
+