UPDATE: load to supabase

app.py

@@ -260,7 +260,8 @@ async def loadPDF(vectorstore: str, pdf: UploadFile = File(...)):
     text = extractTextFromPdf(temp_file_path)
     os.remove(temp_file_path)
     dct = {
-        "output": text
+        "output": text,
+        "source": source
     }
     dct = json.dumps(dct, indent = 1).encode("utf-8")
     fileName = createDataSourceName(sourceName = source)
@@ -287,7 +288,8 @@ async def loadImagePDF(vectorstore: str, pdf: UploadFile = File(...)):
     pdf = await pdf.read()
     text = getTextFromImagePDF(pdfBytes=pdf)
     dct = {
-        "output": text
+        "output": text,
+        "source": source
     }
     dct = json.dumps(dct, indent = 1).encode("utf-8")
     fileName = createDataSourceName(sourceName = source)
@@ -317,7 +319,8 @@ async def loadText(addTextConfig: AddText):
     vectorstore, text = addTextConfig.vectorstore, addTextConfig.text
     username, chatbotName = vectorstore.split("$")[1], vectorstore.split("$")[2]
     dct = {
-        "output": text
+        "output": text,
+        "source": "Text"
     }
     dct = json.dumps(dct, indent = 1).encode("utf-8")
     fileName = createDataSourceName(sourceName = "Text")
@@ -383,14 +386,40 @@ async def addQAPairData(addQaPair: AddQAPair):
         }
 
 
+
+class LoadWebsite(BaseModel):
+    vectorstore: str
+    urls: list[str]
+    source: str
+
+
 @app.post("/loadWebURLs")
-async def addWebsite(vectorstore: str, websiteUrls: list[str]):
-    text = extractTextFromUrlList(urls=websiteUrls)    
+async def loadWebURLs(loadWebsite: LoadWebsite):
+    vectorstore, urls, source = loadWebsite.vectorstore, loadWebsite.urls, loadWebsite.source
+    username, chatbotName = vectorstore.split("$")[1], vectorstore.split("$")[2]
+    text = extractTextFromUrlList(urls=urls)    
+    dct = {
+        "output": text,
+        "source": source
+    }
+    dct = json.dumps(dct, indent = 1).encode("utf-8")
+    fileName = createDataSourceName(sourceName = source)
+    response = supabase.storage.from_("ConversAI").upload(file=dct, path=f"{fileName}_data.json")
+    response = (
+        supabase.table("ConversAI_ChatbotDataSources")
+        .insert({"username": username, 
+                 "chatbotName": chatbotName, 
+                 "dataSourceName": fileName, 
+                 "sourceEndpoint": "\loadWebURLs", 
+                 "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"], f"{fileName}_data.json")})
+        .execute()
+    )
     return {
-        "output": text
+        "output": "SUCCESS"
     }
 
 
+
 @app.post("/answerQuery")
 async def answerQuestion(query: str, vectorstore: str, llmModel: str = "llama3-70b-8192"):
     username, chatbotName = vectorstore.split("$")[1], vectorstore.split("$")[2]