File size: 7,475 Bytes
36d73c0
27e6a14
0770449
9ef164b
0770449
 
27e6a14
c18ec7e
d815dea
c18ec7e
 
 
0770449
27e6a14
0770449
 
2a13ed4
c18ec7e
27e6a14
0770449
0b74b4d
1d1dd8d
0770449
27e6a14
0770449
c18ec7e
 
 
 
 
 
27e6a14
 
 
 
 
 
0770449
 
27e6a14
0770449
 
36d73c0
cb776ef
0b74b4d
fe4b5f1
0770449
 
 
 
 
 
0b74b4d
 
0770449
 
 
8f1d4f2
6229292
b34b7d7
1016fdb
8f1d4f2
 
9ef164b
2c3245b
b1f4ef7
 
 
 
2c3245b
 
 
 
 
d13603d
2c3245b
d13603d
2c3245b
 
b1f4ef7
2c3245b
 
 
 
 
5f76223
2c3245b
 
5f76223
2c3245b
 
 
 
 
 
b1f4ef7
2c3245b
 
 
 
 
 
 
 
27e6a14
cb776ef
2c3245b
 
d13603d
cb776ef
2c3245b
 
 
ea4ffd3
 
9ef164b
 
b1f4ef7
9ef164b
b1f4ef7
ea4ffd3
9ef164b
 
 
2ea73cf
b1f4ef7
9ef164b
 
 
 
 
 
 
 
 
 
b1f4ef7
 
70e00d3
b1f4ef7
70e00d3
 
 
 
2ea73cf
70e00d3
b1f4ef7
70e00d3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b1f4ef7
 
 
eb7bd29
177debf
 
 
 
 
 
 
 
8c08762
 
177debf
 
 
f2fde57
 
 
 
 
 
 
 
 
 
 
 
 
177debf
 
8c08762
 
 
 
 
 
 
 
 
177debf
d5863e7
abff149
42b3dea
ba93db8
 
d5863e7
ef75206
d815dea
 
9b024c3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bd32b51
d815dea
ef75206
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
from typing import Any, Dict, Union

import os
import glob
import shutil
import subprocess
import torch

from fastapi import FastAPI, HTTPException, UploadFile, WebSocket, WebSocketDisconnect
from fastapi.staticfiles import StaticFiles

from pydantic import BaseModel

# langchain
from langchain.chains import RetrievalQA
from langchain.embeddings import HuggingFaceInstructEmbeddings
from langchain.callbacks.base import BaseCallbackHandler
from langchain.schema import LLMResult
from langchain.vectorstores import Chroma

from prompt_template_utils import get_prompt_template
from load_models import load_model

from constants import CHROMA_SETTINGS, EMBEDDING_MODEL_NAME, PERSIST_DIRECTORY, MODEL_ID, MODEL_BASENAME, PATH_NAME_SOURCE_DIRECTORY, SHOW_SOURCES

class Predict(BaseModel):
    prompt: str

class Delete(BaseModel):
    filename: str

if torch.backends.mps.is_available():
    DEVICE_TYPE = "mps"
elif torch.cuda.is_available():
    DEVICE_TYPE = "cuda"
else:
    DEVICE_TYPE = "cpu"

EMBEDDINGS = HuggingFaceInstructEmbeddings(model_name=EMBEDDING_MODEL_NAME, model_kwargs={"device": DEVICE_TYPE})
DB = Chroma(persist_directory=PERSIST_DIRECTORY, embedding_function=EMBEDDINGS, client_settings=CHROMA_SETTINGS)
RETRIEVER = DB.as_retriever()

LLM = load_model(device_type=DEVICE_TYPE, model_id=MODEL_ID, model_basename=MODEL_BASENAME, stream=True, callbacks=[])

prompt, memory = get_prompt_template(promptTemplate_type="llama", history=True)

QA = RetrievalQA.from_chain_type(
    llm=LLM,
    chain_type="stuff",
    retriever=RETRIEVER,
    return_source_documents=SHOW_SOURCES,
    chain_type_kwargs={
        "prompt": prompt,
        "memory": memory
    },
)

app = FastAPI(title="homepage-app")
api_app = FastAPI(title="api app")

app.mount("/api", api_app, name="api")
app.mount("/", StaticFiles(directory="static",html = True), name="static")

@api_app.get("/training")
def run_ingest_route():
    global DB
    global RETRIEVER
    global QA

    try:
        if os.path.exists(PERSIST_DIRECTORY):
            try:
                shutil.rmtree(PERSIST_DIRECTORY)
            except OSError as e:
                raise HTTPException(status_code=500, detail=f"Error: {e.filename} - {e.strerror}.")
        else:
            raise HTTPException(status_code=500, detail="The directory does not exist")

        run_langest_commands = ["python", "ingest.py"]

        if DEVICE_TYPE == "cpu":
            run_langest_commands.append("--device_type")
            run_langest_commands.append(DEVICE_TYPE)

        result = subprocess.run(run_langest_commands, capture_output=True)

        if result.returncode != 0:
            raise HTTPException(status_code=400, detail="Script execution failed: {}")

        # load the vectorstore
        DB = Chroma(
            persist_directory=PERSIST_DIRECTORY,
            embedding_function=EMBEDDINGS,
            client_settings=CHROMA_SETTINGS,
        )

        RETRIEVER = DB.as_retriever()

        QA = RetrievalQA.from_chain_type(
            llm=LLM,
            chain_type="stuff",
            retriever=RETRIEVER,
            return_source_documents=SHOW_SOURCES,
            chain_type_kwargs={
                "prompt": prompt,
                "memory": memory
            },
        )

        return {"response": "The training was successfully completed"}
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Error occurred: {str(e)}")

@api_app.get("/api/files")
def get_files():
    upload_dir = os.path.join(os.getcwd(), PATH_NAME_SOURCE_DIRECTORY)
    files = glob.glob(os.path.join(upload_dir, '*'))

    return {"directory": upload_dir, "files": files}

@api_app.delete("/api/delete_document")
def delete_source_route(data: Delete):
    filename = data.filename
    path_source_documents = os.path.join(os.getcwd(), PATH_NAME_SOURCE_DIRECTORY)
    file_to_delete = f"{path_source_documents}/{filename}"

    if os.path.exists(file_to_delete):
        try:
            os.remove(file_to_delete)
            print(f"{file_to_delete} has been deleted.")

            return {"message": f"{file_to_delete} has been deleted."}
        except OSError as e:
            raise HTTPException(status_code=400, detail=print(f"error: {e}."))
    else:
         raise HTTPException(status_code=400, detail=print(f"The file {file_to_delete} does not exist."))

@api_app.post('/predict')
def predict(data: Predict):
    global QA
    try:
        user_prompt = data.prompt
        if user_prompt:
            res = QA(user_prompt)

            answer, docs = res["result"], res["source_documents"]

            prompt_response_dict = {
                "Prompt": user_prompt,
                "Answer": answer,
            }

            prompt_response_dict["Sources"] = []
            for document in docs:
                prompt_response_dict["Sources"].append(
                    (os.path.basename(str(document.metadata["source"])), str(document.page_content))
                )

            return {"response": prompt_response_dict}
        else:
            raise HTTPException(status_code=400, detail="Prompt Incorrect")
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Error occurred: {str(e)}")



@api_app.post("/save_document/")
async def create_upload_file(file: UploadFile):
    # Get the file size (in bytes)
    file.file.seek(0, 2)
    file_size = file.file.tell()

    # move the cursor back to the beginning
    await file.seek(0)

    if file_size > 10 * 1024 * 1024:
        # more than 10 MB
        raise HTTPException(status_code=400, detail="File too large")

    content_type = file.content_type

    if content_type not in [
        "text/plain",
        "text/markdown",
        "text/x-markdown",
        "text/csv",
        "application/msword",
        "application/pdf",
        "application/vnd.ms-excel",
        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
        "text/x-python",
        "application/x-python-code"]:
        raise HTTPException(status_code=400, detail="Invalid file type")

    upload_dir = os.path.join(os.getcwd(), PATH_NAME_SOURCE_DIRECTORY)
    if not os.path.exists(upload_dir):
        os.makedirs(upload_dir)

    dest = os.path.join(upload_dir, file.filename)

    with open(dest, "wb") as buffer:
        shutil.copyfileobj(file.file, buffer)

    return {"filename": file.filename}

@api_app.websocket("/ws/{client_id}")
async def websocket_endpoint(websocket: WebSocket,  client_id: str):
    global QA

    await websocket.accept()

    try:
        while True:
            user_prompt = await websocket.receive_text()
            response = QA(inputs=user_prompt, return_only_outputs=True, tags=f'{client_id}', include_run_info=True)
            answer, docs = response["result"], response["source_documents"]

            prompt_response_dict = {
                "Prompt": user_prompt,
                "Answer": answer,
            }

            prompt_response_dict["Sources"] = []
            for document in docs:
                prompt_response_dict["Sources"].append(
                    (os.path.basename(str(document.metadata["source"])), str(document.page_content))
                )
            await websocket.send_json(prompt_response_dict)

    except WebSocketDisconnect:
        print('disconnect')
    except RuntimeError as error:
        print(error)