Upload 4 files

Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.11-slim
+
+WORKDIR /usr/src/app
+
+COPY requirements.txt ./
+RUN pip install --upgrade pip && \
+    pip install -r requirements.txt &&\
+    pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
+
+COPY ./app ./app
+
+# Copy the model files into the Docker image
+COPY flan-t5-small /models/flan-t5-small
+
+EXPOSE 5005
+
+CMD ["uvicorn", "app.server:app", "--host", "0.0.0.0", "--port", "5005"]

app/llama2.py

@@ -0,0 +1,22 @@
+from transformers import pipeline, LlamaForCausalLM, LlamaTokenizer
+from langchain_community.llms.huggingface_pipeline import HuggingFacePipeline
+
+# Set the model path to the location where your model files are stored
+model_path = "/models/flan-t5-small"  # Adjust this path as needed
+
+# Load the model and tokenizer
+local_model = LlamaForCausalLM.from_pretrained(model_path, return_dict=True)
+local_tokenizer = LlamaTokenizer.from_pretrained(model_path)
+
+# Create a text generation pipeline
+pipe = pipeline(
+    task="text-generation",
+    model=local_model,
+    tokenizer=local_tokenizer,
+    max_new_tokens=100,
+    repetition_penalty=1.1,
+    model_kwargs={"max_length": 1200, "temperature": 0.01}
+)
+
+# Pipeline to be consumed by Langserve API
+llm_pipeline = HuggingFacePipeline(pipeline=pipe)

app/server.py

@@ -0,0 +1,40 @@
+from fastapi import FastAPI
+from langchain.prompts import PromptTemplate 
+from fastapi.responses import RedirectResponse
+from fastapi.middleware.cors import CORSMiddleware
+from langserve import add_routes
+from app.llama2 import llm_pipeline
+
+app = FastAPI()
+
+# Set up CORS middleware to allow requests from any origin
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],  # Set this to the specific origin of your frontend in production
+    allow_credentials=False,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+template = """You are a very smart and educated assistant to guide the user to understand the concepts. Please Explaining the answer
+If you don't know the answer, just say that you don't know, don't try to make up an answer.
+
+Question: {question}
+
+Only return the helpful answer below and nothing else. Give an answer in 1000 characteres at maximum please
+Helpful answer:
+"""
+
+prompt = PromptTemplate.from_template(template) 
+
+@app.get("/")
+async def redirect_root_to_docs():
+    return RedirectResponse("/docs")
+
+add_routes(app, 
+           prompt|llm_pipeline,
+           path='/chain_llama_non')
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="localhost", port=5005)

requirements.txt

@@ -0,0 +1,9 @@
+transformers
+langchain_community
+sentencepiece
+langserve
+langchain
+fastapi
+pydantic==1.10.13
+uvicorn
+sse_starlette