File size: 8,502 Bytes
0770449
9ef164b
0770449
 
27e6a14
5d66516
4965a65
 
 
 
c18ec7e
d815dea
c18ec7e
5d66516
c18ec7e
 
0770449
27e6a14
0770449
 
2a13ed4
c18ec7e
27e6a14
0770449
0b74b4d
1d1dd8d
0770449
911d720
0770449
c18ec7e
 
 
 
 
 
9f4aea3
 
 
 
 
 
 
 
0770449
 
27e6a14
0770449
 
b0d4d1d
cb776ef
b663f39
fe4b5f1
0770449
 
 
 
 
 
0b74b4d
0770449
 
 
5073361
d2ccd1f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5d66516
 
8f1d4f2
6229292
b34b7d7
1016fdb
8f1d4f2
 
9ef164b
2c3245b
b1f4ef7
 
 
 
2c3245b
 
 
 
 
d13603d
2c3245b
d13603d
2c3245b
 
b1f4ef7
9f4aea3
 
 
2c3245b
 
5f76223
2c3245b
 
5f76223
2c3245b
 
 
 
 
 
b1f4ef7
2c3245b
 
 
 
 
 
 
 
4965a65
2c3245b
 
d13603d
cb776ef
2c3245b
 
 
ea4ffd3
 
9ef164b
 
b1f4ef7
9ef164b
b1f4ef7
ea4ffd3
9ef164b
 
 
2ea73cf
b1f4ef7
9ef164b
 
 
 
 
 
 
 
 
 
b1f4ef7
 
70e00d3
b1f4ef7
70e00d3
 
 
d2ccd1f
70e00d3
 
 
 
 
 
b1f4ef7
eb7bd29
177debf
 
 
 
 
 
 
 
8c08762
 
177debf
 
 
f2fde57
 
 
 
 
 
 
 
 
 
 
 
 
177debf
 
8c08762
 
 
 
 
 
 
 
 
177debf
d5863e7
9117ff3
 
b0d4d1d
598b96c
5d66516
9117ff3
5d66516
 
9117ff3
 
ef75206
d815dea
 
5d66516
9b024c3
d2ccd1f
5d66516
9117ff3
bd32b51
d815dea
9117ff3
5d66516
 
9117ff3
5d66516
 
9117ff3
ef75206
 
48c871b
9117ff3
 
b0d4d1d
598b96c
48c871b
9117ff3
48c871b
 
f1368ae
9117ff3
48c871b
 
 
 
 
b21e4ba
48c871b
9117ff3
48c871b
 
9117ff3
48c871b
 
9117ff3
48c871b
 
9117ff3
48c871b
 
9117ff3
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
import os
import glob
import shutil
import subprocess
import torch
import json
import faulthandler

faulthandler.enable()


from fastapi import FastAPI, HTTPException, UploadFile, WebSocket, WebSocketDisconnect
from fastapi.staticfiles import StaticFiles
from websocket.socketManager import WebSocketManager

from pydantic import BaseModel

# langchain
from langchain.chains import RetrievalQA
from langchain.embeddings import HuggingFaceInstructEmbeddings
from langchain.callbacks.base import BaseCallbackHandler
from langchain.schema import LLMResult
from langchain.vectorstores import Chroma

from prompt_template_utils import get_prompt_template
from load_models import load_model

from constants import CHROMA_SETTINGS, EMBEDDING_MODEL_NAME, PERSIST_DIRECTORY, MODEL_ID, MODEL_BASENAME, PATH_NAME_SOURCE_DIRECTORY, SHOW_SOURCES, CONTEXT_WINDOW_SIZE, MAX_NEW_TOKENS

class Predict(BaseModel):
    prompt: str

class Delete(BaseModel):
    filename: str

# if torch.backends.mps.is_available():
#     DEVICE_TYPE = "mps"
# elif torch.cuda.is_available():
#     DEVICE_TYPE = "cuda"
# else:
#     DEVICE_TYPE = "cpu"

DEVICE_TYPE = "cuda"

EMBEDDINGS = HuggingFaceInstructEmbeddings(model_name=EMBEDDING_MODEL_NAME, model_kwargs={"device": DEVICE_TYPE})
DB = Chroma(persist_directory=PERSIST_DIRECTORY, embedding_function=EMBEDDINGS, client_settings=CHROMA_SETTINGS)
RETRIEVER = DB.as_retriever()

LLM = load_model(device_type=DEVICE_TYPE, model_id=MODEL_ID, model_basename=MODEL_BASENAME, stream=True)

prompt, memory = get_prompt_template(promptTemplate_type="llama", history=False)

QA = RetrievalQA.from_chain_type(
    llm=LLM,
    chain_type="stuff",
    retriever=RETRIEVER,
    return_source_documents=SHOW_SOURCES,
    chain_type_kwargs={
        "prompt": prompt,
    },
)

def sendPromptChain(QA, user_prompt):
    res = QA(user_prompt)

    answer, docs = res["result"], res["source_documents"]

    prompt_response_dict = {
        "Prompt": user_prompt,
        "Answer": answer,
    }

    prompt_response_dict["Sources"] = []
    for document in docs:
        prompt_response_dict["Sources"].append(
            (os.path.basename(str(document.metadata["source"])), str(document.page_content))
        )

    return prompt_response_dict;

socket_manager = WebSocketManager()

app = FastAPI(title="homepage-app")
api_app = FastAPI(title="api app")

app.mount("/api", api_app, name="api")
app.mount("/", StaticFiles(directory="static",html = True), name="static")

@api_app.get("/training")
def run_ingest_route():
    global DB
    global RETRIEVER
    global QA

    try:
        if os.path.exists(PERSIST_DIRECTORY):
            try:
                shutil.rmtree(PERSIST_DIRECTORY)
            except OSError as e:
                raise HTTPException(status_code=500, detail=f"Error: {e.filename} - {e.strerror}.")
        else:
            raise HTTPException(status_code=500, detail="The directory does not exist")

        run_langest_commands = ["python", "ingest.py"]

        # if DEVICE_TYPE == "cpu":
        #     run_langest_commands.append("--device_type")
        #     run_langest_commands.append(DEVICE_TYPE)

        result = subprocess.run(run_langest_commands, capture_output=True)

        if result.returncode != 0:
            raise HTTPException(status_code=400, detail="Script execution failed: {}")

        # load the vectorstore
        DB = Chroma(
            persist_directory=PERSIST_DIRECTORY,
            embedding_function=EMBEDDINGS,
            client_settings=CHROMA_SETTINGS,
        )

        RETRIEVER = DB.as_retriever()

        QA = RetrievalQA.from_chain_type(
            llm=LLM,
            chain_type="stuff",
            retriever=RETRIEVER,
            return_source_documents=SHOW_SOURCES,
            chain_type_kwargs={
                "prompt": prompt
            },
        )

        return {"response": "The training was successfully completed"}
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Error occurred: {str(e)}")

@api_app.get("/api/files")
def get_files():
    upload_dir = os.path.join(os.getcwd(), PATH_NAME_SOURCE_DIRECTORY)
    files = glob.glob(os.path.join(upload_dir, '*'))

    return {"directory": upload_dir, "files": files}

@api_app.delete("/api/delete_document")
def delete_source_route(data: Delete):
    filename = data.filename
    path_source_documents = os.path.join(os.getcwd(), PATH_NAME_SOURCE_DIRECTORY)
    file_to_delete = f"{path_source_documents}/{filename}"

    if os.path.exists(file_to_delete):
        try:
            os.remove(file_to_delete)
            print(f"{file_to_delete} has been deleted.")

            return {"message": f"{file_to_delete} has been deleted."}
        except OSError as e:
            raise HTTPException(status_code=400, detail=print(f"error: {e}."))
    else:
         raise HTTPException(status_code=400, detail=print(f"The file {file_to_delete} does not exist."))

@api_app.post('/predict')
def predict(data: Predict):
    global QA
    try:
        user_prompt = data.prompt
        if user_prompt:
            prompt_response_dict = sendPromptChain(QA, user_prompt)

            return {"response": prompt_response_dict}
        else:
            raise HTTPException(status_code=400, detail="Prompt Incorrect")
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Error occurred: {str(e)}")

@api_app.post("/save_document/")
async def create_upload_file(file: UploadFile):
    # Get the file size (in bytes)
    file.file.seek(0, 2)
    file_size = file.file.tell()

    # move the cursor back to the beginning
    await file.seek(0)

    if file_size > 10 * 1024 * 1024:
        # more than 10 MB
        raise HTTPException(status_code=400, detail="File too large")

    content_type = file.content_type

    if content_type not in [
        "text/plain",
        "text/markdown",
        "text/x-markdown",
        "text/csv",
        "application/msword",
        "application/pdf",
        "application/vnd.ms-excel",
        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
        "text/x-python",
        "application/x-python-code"]:
        raise HTTPException(status_code=400, detail="Invalid file type")

    upload_dir = os.path.join(os.getcwd(), PATH_NAME_SOURCE_DIRECTORY)
    if not os.path.exists(upload_dir):
        os.makedirs(upload_dir)

    dest = os.path.join(upload_dir, file.filename)

    with open(dest, "wb") as buffer:
        shutil.copyfileobj(file.file, buffer)

    return {"filename": file.filename}

@api_app.websocket("/ws/{user_id}")
async def websocket_endpoint_student(websocket: WebSocket, user_id: str):
    global QA

    message = {
        "message": f"Student {user_id} connected"
    }

    await socket_manager.add_user_to_room(user_id, websocket)
    await socket_manager.broadcast_to_room(user_id, json.dumps(message))

    try:
        while True:
            data = await websocket.receive_text()

            prompt_response_dict = sendPromptChain(QA, data)

            await socket_manager.broadcast_to_room(user_id, json.dumps(prompt_response_dict))

    except WebSocketDisconnect:
        await socket_manager.remove_user_from_room(user_id, websocket)

        message = {
            "message": f"Student {user_id} disconnected"
        }

        await socket_manager.broadcast_to_room(user_id, json.dumps(message))
    except RuntimeError as error:
        print(error)

@api_app.websocket("/ws/{room_id}/{user_id}")
async def websocket_endpoint_room(websocket: WebSocket, room_id: str, user_id: str):
    global QA

    message = {
        "message": f"Student {user_id} connected to the classroom"
    }

    await socket_manager.add_user_to_room(room_id, websocket)
    await socket_manager.broadcast_to_room(room_id, json.dumps(message))

    try:
        while True:
            data = await websocket.receive_text()

            prompt_response_dict = sendPromptChain(QA, data)

            await socket_manager.broadcast_to_room(room_id, json.dumps(prompt_response_dict))

    except WebSocketDisconnect:
        await socket_manager.remove_user_from_room(room_id, websocket)

        message = {
            "message": f"Student {user_id} disconnected from room - {room_id}"
        }

        await socket_manager.broadcast_to_room(room_id, json.dumps(message))
    except RuntimeError as error:
        print(error)