creating embedding from docs

app.py

@@ -4,7 +4,7 @@ import logging
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.embeddings.openai import OpenAIEmbeddings
 import chainlit as cl
-from src.utils import get_docSearch
+from src.utils import get_docSearch, get_source
 from src.model import load_chain
 
 
@@ -13,11 +13,13 @@ from src.model import load_chain
 
 
 
+
 welcome_message = """ Upload your file here"""
 
 @cl.on_chat_start
 async def start():
     await cl.Message("you are in ").send()
+    logging.info(f"app started")
     files = None
     while files is None:
         files = await cl.AskFileMessage(
@@ -26,12 +28,16 @@ async def start():
             max_size_mb=10,
             timeout=90
         ).send()
+    logging.info("uploader excecuted")
     file = files[0]
     msg = cl.Message(content=f"Processing `{type(files)}` {file.name}....")
     await msg.send()
 
-    docsearch = get_docSearch(file)
+    logging.info("processing started")
+
+    docsearch = get_docSearch(file,cl)
     
+    logging.info("document uploaded success")
 
     chain = load_chain(docsearch)
 
@@ -44,6 +50,34 @@ async def start():
 
     await msg.update()
 
+    logging.info("processing completed")
+
     cl.user_session.set("chain", chain)
 
+@cl.on_message
+async def main(message):
+    chain = cl.user_session.get("chain")
+    cb = cl.AsyncLangchainCallbackHandler(
+        stream_final_answer=True, answer_prefix_tokens=["FINAL","ANSWER"]
+    )
+
+    cb.answer_reached = True
+    res = await chain.acall(message, callbacks=[cb])
+
+    answer = res["answer"]
+    sources = res["sources"].strip()
+    
+
+    ## get doc from user session
+    docs = cl.user_session.get("docs")
+    metadatas = [doc.metadata for doc in docs]
+    all_sources = [m["source"]for m in metadatas]
+
+    source_elements,answer = get_source(sources,all_sources,docs,cl)
+
+    if cb.has_streamed_final_answer:
+        cb.final_stream.elements = source_elements
+        await cb.final_stream.update()
+    else:
+        await cl.Message(content=answer, elements=source_elements).send()
 

requirements.txt

@@ -3,4 +3,5 @@ openai
 python-dotenv
 chainlit
 chromadb
-tiktoken
+tiktoken
+tokenizers

src/config.py

@@ -1,5 +1,13 @@
+import os
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+
+
 class Config:
     temperature = 0
     streaming = True
     chain_type = "stuff"
-    max_token_limit = 4098
+    max_token_limit = 4098
+    embeddings = OpenAIEmbeddings(api_key=os.getenv('OPENAI_API_KEY'))
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)

src/model.py

@@ -1,7 +1,7 @@
 from langchain.chains import RetrievalQAWithSourcesChain
 from langchain.chat_models import ChatOpenAI
 import logging
-
+import os
 
 
 from src.config import Config
@@ -12,7 +12,7 @@ from src.config import Config
 
 def load_model():
     model = ChatOpenAI(temperature=Config.temperature,
-                   streaming=Config.streaming)
+                   streaming=Config.streaming,api_key=os.getenv('OPENAI_API_KEY'))
     return model
 
 

src/utils.py

@@ -1,12 +1,21 @@
 from chainlit.types import AskFileResponse
+import click
 from langchain.document_loaders import TextLoader
 from langchain.document_loaders import PyPDFDirectoryLoader
-from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.vectorstores import Chroma
-from langchain.embeddings import OpenAIEmbeddings
 
-text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
-embeddings = OpenAIEmbeddings()
+
+from src.config import Config
+# import chainlit as cl
+import logging
+import openai
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+
 
 def process_file(file: AskFileResponse):
     import tempfile 
@@ -21,17 +30,49 @@ def process_file(file: AskFileResponse):
         loader = Loader(tempfile.name)
         documents = loader.load()
         # text_splitter = text_splitter()
-        docs = text_splitter.split_documents(documents)
+        docs = Config.text_splitter.split_documents(documents)
 
         for i, doc in enumerate(docs):
             doc.metadata["source"] = f"source_{i}"
         return docs
 
-def get_docSearch(file: AskFileResponse):
+def get_docSearch(file,cl):
     docs = process_file(file)
 
+    logging.info("files loaded ")
+
     ## save data in user session 
+    cl.user_session.set("docs",docs)
+
+    logging.info("docs saved in active session")
     
-    docsearch = Chroma.from_documents(docs, embeddings)
+    docsearch = Chroma.from_documents(docs, Config.embeddings)
+
+    logging.info("embedding completed")
+
+    return docsearch
+
+def get_source(sources,all_sources,docs,cl):
+    answer = []
+    source_elements = []
+    if sources:
+        found_sources = []
+
+        # Add the sources to the message
+        for source in sources.split(","):
+            source_name = source.strip().replace(".", "")
+            # Get the index of the source
+            try:
+                index = all_sources.index(source_name)
+            except ValueError:
+                continue
+            text = docs[index].page_content
+            found_sources.append(source_name)
+            # Create the text element referenced in the message
+            source_elements.append(cl.Text(content=text, name=source_name))
 
-    return docsearch
+        if found_sources:
+            answer += f"\nSources: {', '.join(found_sources)}"
+        else:
+            answer += "\nNo sources found"
+    return source_elements,answer