Initial commit

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.pdf filter=lfs diff=lfs merge=lfs -text
+vectorstores/db_faiss/index.faiss filter=lfs diff=lfs merge=lfs -text
+

Dockerfile

@@ -1,16 +1,31 @@
-# read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
-# you will also find guides on how best to write your Dockerfile
-
-FROM python:3.9
+# Use the official Python image as base
+FROM python:3.9-slim
 
+# Create a non-root user
 RUN useradd -m -u 1000 user
 
+# Set the working directory in the container
 WORKDIR /app
 
+# Install system dependencies
+RUN apt-get update && \
+    apt-get install -y gcc python3-dev python3-venv && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Copy requirements file
 COPY --chown=user ./requirements.txt requirements.txt
 
-RUN pip install --no-cache-dir --upgrade -r requirements.txt
+# Install dependencies
+RUN python3 -m venv env && \
+    /bin/bash -c "source env/bin/activate && pip install --no-cache-dir --upgrade -r requirements.txt" && \
+    /bin/bash -c "source env/bin/activate && pip install chainlit langchain_community"
+
+# Make port 7680 available to the world outside the container
+EXPOSE 7680
 
+# Copy the application code into the container
 COPY --chown=user . /app
 
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
+# Run the application
+CMD ["/bin/bash", "-c", "source env/bin/activate && uvicorn app:app --host 0.0.0.0 --port 7680 && python3 downloadLLM.py && python3 ingest.py && chainlit run app.py --host 0.0.0.0 --port 7680"]

app.py

@@ -1,7 +1,32 @@
-from fastapi import FastAPI
 
-app = FastAPI()
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.document_loaders import PyPDFLoader, DirectoryLoader # could have done any unstructured text loader like ppt and xlsx
 
-@app.get("/")
-def greet_json():
-    return {"Hello": "World!"}
+
+from langchain.embeddings import HuggingFaceBgeEmbeddings # we can replace huggingface with facetransformers
+from chainlit import cl
+from langchain.vectorstores import FAISS
+
+DATA_PATH = "data/"
+DB_FAISS_PATH = "vectorstores/db_faiss"
+
+#create vector database
+def create_vector_db():
+                                            # WE can change .pdf with any other unstructured text format
+    loader = DirectoryLoader(DATA_PATH, glob="*.pdf", loader_cls = PyPDFLoader)
+    documents = loader.load()
+
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=50)
+
+    texts = text_splitter.split_documents(documents)
+
+    embeddings = HuggingFaceBgeEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2", model_kwargs={"device": "cpu"}) # change to GPU if you want
+
+    # cuda is not supported in my MAC M1! SADLY.
+
+    db = FAISS.from_documents(texts, embeddings)
+    db.save_local(DB_FAISS_PATH)
+
+if __name__ == "__main__":
+    create_vector_db()
+    cl.run()

chainlit.md

@@ -0,0 +1,28 @@
+# AskAway - HackUTA 2023 🚀🚀🚀
+This project was built for HackUTA 2023 to demonstrate an LLM which can  be trained and ran offline on local host. 
+
+## Developers
+ -**Alok Jha**
+ -**Ribesh Joshi**
+ -**Khushi Gauli**
+ -**Kritazya Upreti**
+
+
+## About the project
+
+This project showcases how a large language model can be fine-tuned on a custom dataset and deployed for low-latency inference completely offline on a local machine.
+
+The project uses the LangChain library to handle data ingestion, model training loops and managing the model lifecycle. The ChainLit framework is used to package and deploy the LLM inference server for easy access through a web UI or APIs.
+
+## Some key aspects:
+-**Fine-tunes a small-sized LLM like MiniLM for custom data**
+-**Does not require an internet connection after deployment**
+-**Low-latency querying against the fine-tuned LLM**
+-**Easy to update model as new data comes in**
+-**Python implementation allows full customization**
+
+This demonstrates how modern transfer learning techniques can be used to take a pre-trained LLM and specialize it for niche domains and offline usage.
+
+## Usage
+-**Give the LLM domain-specific data**
+-**Ask natural language questions to get information about the data**

data/anatomy+phys+vol2a.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e3365119163aa2501b4ef0f57cf498f63ef4e3a1b9bc1cd4833a293499f3efa
+size 48242701

dcoker commands.txt

@@ -0,0 +1,32 @@
+docker build -t anatomyai .
+docker run -p 8000:8000 anatomyai
+
+
+  Push Your Docker Image to Docker Hub
+Log in to Docker Hub:
+
+bash
+Copy code
+docker login
+Tag your Docker image:
+
+bash
+Copy code
+docker tag anatomyai your_dockerhub_username/anatomyai:latest
+Push your Docker image:
+
+bash
+Copy code
+docker push your_dockerhub_username/anatomyai:latest
+Deploy to Hugging Face
+Create a new Space on Hugging Face by going to the Spaces page and clicking on "New Space".
+
+Choose the Docker runtime for your Space.
+
+Configure your Space:
+
+In the Space settings, provide the name of your Docker image (your_dockerhub_username/anatomyai:latest).
+Set any necessary environment variables or secrets required by your application.
+Deploy the Space:
+
+Once you have configured the Space, Hugging Face will pull the Docker image from Docker Hub and deploy it.

dcoker mac

@@ -0,0 +1,33 @@
+# Use an official Python runtime as a parent image
+FROM python:3.9-slim
+
+# Set the working directory in the container
+WORKDIR /app
+
+# Install necessary packages
+RUN apt-get update && \
+    apt-get install -y gcc python3-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Copy the current directory contents into the container at /app
+COPY . /app
+
+# Install virtualenv
+RUN pip install virtualenv
+
+# Create and activate virtual environment, install dependencies, and Chainlit
+RUN python3 -m venv env && \
+    /bin/bash -c "source env/bin/activate && \
+    pip install --no-cache-dir -r requirements.txt && \
+    pip install chainlit && \
+    pip install langchain_community"
+
+# Make port 8000 available to the world outside this container
+EXPOSE 8000
+
+# Run your Python scripts and Chainlit app
+CMD /bin/bash -c "source env/bin/activate && \
+    python3 downloadLLM.py && \
+    python3 ingest.py && \
+    chainlit run main.py --host 0.0.0.0 --port 8000"

downloadLLM.py

@@ -0,0 +1,19 @@
+import requests
+
+url = "https://cdn-lfs.huggingface.co/repos/30/e3/30e3aca7233f7337633262ff6d59dd98559ecd8982e7419b39752c8d0daae1ca/8daa9615cce30c259a9555b1cc250d461d1bc69980a274b44d7eda0be78076d8?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27llama-2-7b-chat.ggmlv3.q4_0.bin%3B+filename%3D%22llama-2-7b-chat.ggmlv3.q4_0.bin%22%3B&response-content-type=application%2Foctet-stream&Expires=1717963444&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcxNzk2MzQ0NH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9yZXBvcy8zMC9lMy8zMGUzYWNhNzIzM2Y3MzM3NjMzMjYyZmY2ZDU5ZGQ5ODU1OWVjZDg5ODJlNzQxOWIzOTc1MmM4ZDBkYWFlMWNhLzhkYWE5NjE1Y2NlMzBjMjU5YTk1NTViMWNjMjUwZDQ2MWQxYmM2OTk4MGEyNzRiNDRkN2VkYTBiZTc4MDc2ZDg%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qJnJlc3BvbnNlLWNvbnRlbnQtdHlwZT0qIn1dfQ__&Signature=SyIt4jnVZLhKHXELakpuirrJy9HHG7ETsYGWP0u8KhmKD2sv8hY35khc0F7fOG6Gh9sdSqvebSzFz-RrqMw-ibcLmzqoRJ35ZKNE0JjMmk61APtW0DiMMD8bCwRHQs8T3IA-eIP4ybz06UD3NTVTTrKCGkPw8nZMImph-BFmx6fOO9JD9CrrQ7TBE-LOAbfsGzOF-nAXjPhGzYvtliIsATipqDkTgGKOcJx9PeQDyHBRaHHr5jrss20%7EUixaWRHAt6Og2JJUA7CK%7ElCEy7Jgo5--%7EqDZoyfXxhvV6zqsaZrs1aXxoCov-QBoz6hEN5yKPjpWJC9DYjMJn4kce3o9fQ__&Key-Pair-Id=KVTP0A1DKRTAX"
+
+file_name = "llama-2-7b-chat.ggmlv3.q4_0.bin"
+
+# Send GET request to the URL
+response = requests.get(url, stream=True)
+
+# Raise an exception in case of an HTTP error
+response.raise_for_status()
+
+# Write the content of the response to a file
+with open(file_name, 'wb') as file:
+    for chunk in response.iter_content(chunk_size=8192):
+        if chunk:
+            file.write(chunk)
+print("Downloading llm")
+print(f"Downloaded 'llama-2-7b-chat.ggmlv3.q4_0.bin' successfully.")

ingest.py

@@ -0,0 +1,30 @@
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.document_loaders import PyPDFLoader, DirectoryLoader # could have done any unstructured text loader like ppt and xlsx
+
+
+from langchain.embeddings import HuggingFaceBgeEmbeddings # we can replace huggingface with facetransformers
+
+from langchain.vectorstores import FAISS
+
+DATA_PATH = "data/"
+DB_FAISS_PATH = "vectorstores/db_faiss"
+
+#create vector database
+def create_vector_db():
+                                            # WE can change .pdf with any other unstructured text format
+    loader = DirectoryLoader(DATA_PATH, glob="*.pdf", loader_cls = PyPDFLoader)
+    documents = loader.load()
+
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
+
+    texts = text_splitter.split_documents(documents)
+
+    embeddings = HuggingFaceBgeEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2", model_kwargs={"device": "cpu"}) # change to GPU if you want
+
+    # cuda is not supported in my MAC M1! SADLY.
+
+    db = FAISS.from_documents(texts, embeddings)
+    db.save_local(DB_FAISS_PATH)
+
+if __name__ == "__main__":
+    create_vector_db()

main.py

@@ -0,0 +1,31 @@
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.document_loaders import PyPDFLoader, DirectoryLoader # could have done any unstructured text loader like ppt and xlsx
+
+
+from langchain.embeddings import HuggingFaceBgeEmbeddings # we can replace huggingface with facetransformers
+from chainlit import cl
+from langchain.vectorstores import FAISS
+
+DATA_PATH = "data/"
+DB_FAISS_PATH = "vectorstores/db_faiss"
+
+#create vector database
+def create_vector_db():
+                                            # WE can change .pdf with any other unstructured text format
+    loader = DirectoryLoader(DATA_PATH, glob="*.pdf", loader_cls = PyPDFLoader)
+    documents = loader.load()
+
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=50)
+
+    texts = text_splitter.split_documents(documents)
+
+    embeddings = HuggingFaceBgeEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2", model_kwargs={"device": "cpu"}) # change to GPU if you want
+
+    # cuda is not supported in my MAC M1! SADLY.
+
+    db = FAISS.from_documents(texts, embeddings)
+    db.save_local(DB_FAISS_PATH)
+
+if __name__ == "__main__":
+    create_vector_db()
+    cl.run()

model.py

@@ -0,0 +1,154 @@
+from langchain import PromptTemplate
+from langchain.embeddings import HuggingFaceBgeEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.llms import ctransformers
+from langchain.chains import RetrievalQA
+
+import chainlit as cl
+
+
+DB_FAISS_PATH = "vectorstores/db_faiss"
+
+
+input_variables = ["Context", "question"]
+
+
+custom_prompt_template = """Use the following pieces of information to answer the user's question.
+
+
+Context = {Context}
+
+Question = {question}
+
+
+
+only returns the helpful answer below and nothing else
+
+Helpful answer
+
+"""
+
+
+def set_custon_prompt():
+
+    prompt = PromptTemplate(template=custom_prompt_template,
+
+                            input_variables=input_variables,
+
+                            validate_variable_names=False)
+
+    return prompt
+
+
+def load_llm():
+
+    llm = ctransformers.CTransformers(
+
+        # old model = llama-2-7b-chat.ggmlv3.q8_0.bin
+        model="llama-2-7b-chat.ggmlv3.q4_0.bin",
+
+        model_type='llama',
+
+        max_new_tokens=512,
+
+        temperature=0.5
+
+    )
+
+    return llm
+
+
+def retrival_qa_chain(llm, prompt, db):
+
+    qa_chain = RetrievalQA.from_chain_type(
+
+        llm=llm,
+
+        chain_type="stuff",
+
+        retriever=db.as_retriever(search_kwargs={"k": 2}),
+
+
+
+        return_source_documents=True,
+
+        chain_type_kwargs={'prompt': prompt, 'document_variable_name': 'Context'}
+
+    )
+
+    return qa_chain
+
+
+def qa_bot():
+
+    embeddings = HuggingFaceBgeEmbeddings(
+        model_name="sentence-transformers/all-MiniLM-L6-v2", model_kwargs={"device": "cpu"})
+
+    db = FAISS.load_local(DB_FAISS_PATH, embeddings)
+
+    llm = load_llm()
+
+    qa_prompt = set_custon_prompt()
+
+    qa = retrival_qa_chain(llm, qa_prompt, db)
+
+    return qa
+
+
+def final_result(query):
+
+    qa_result = qa_bot()
+
+    response = qa_result({'query': query})
+
+    return response
+
+
+### Chain LIT ###
+
+@cl.on_chat_start
+async def start():
+
+    chain = qa_bot()
+
+    cl.user_session.set("chain", chain)
+
+    msg = cl.Message(content="Starting the bot.....")
+
+    await msg.send()
+
+    msg.content = "Hi, What is your query?"
+
+    await msg.update()
+
+
+@cl.on_message
+async def main(message):
+
+    chain = cl.user_session.get("chain")
+
+    cb = cl.AsyncLangchainCallbackHandler(
+
+        stream_final_answer=True, answer_prefix_tokens=["FINAL", "ANSWER"]
+
+
+
+    )
+
+    cb.answer_reached = True
+
+    res = await chain.acall(message, callbacks=[cb])
+
+    answer = res['result']
+
+    sources = res['source_documents']
+
+    if sources:
+
+        answer += f"\nSources: " + str(sources)
+
+    else:
+
+        answer += f"\nSources: No sources found"
+
+    await cl.Message(content=answer).send()

package.json

@@ -0,0 +1,8 @@
+{
+    "name": "chainlit-app",
+    "version": "1.0.0",
+    "scripts": {
+      "build": "sh build.sh"
+    }
+  }
+  

requirements.txt

@@ -1,2 +1,11 @@
-fastapi
-uvicorn[standard]
+pypdf
+langchain
+torch
+accelerate
+bitsandbytes
+transformers
+sentence_transformers
+faiss_cpu
+chainlit
+langchain_community
+gdown