update in code.

app.py

@@ -5,21 +5,32 @@ try:
     import langchain
     from langchain import Cohere
     import os
-    from langchain.document_loaders import PagedPDFSplitter
     from langchain.embeddings.cohere import CohereEmbeddings
     from langchain.vectorstores import Chroma
     from langchain.text_splitter import CharacterTextSplitter
     from langchain.llms import Cohere
+    from langchain.chains import ChatVectorDBChain
+    from langchain.text_splitter import RecursiveCharacterTextSplitter
 except Exception as  ex:
     print(f'Some Error --> {ex}')
 
-doclist=[doc for doc in [docs for docs in list(map(lambda pdffile : PagedPDFSplitter(pdffile).load_and_split(),list(map(lambda fileref:os.path.abspath(f'data/{fileref}'),os.listdir('./data')))))]]    
-flat_list = [item for sublist in doclist for item in sublist]
-text_splitter=CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
-docs=text_splitter.split_documents(flat_list)
+with open('kb.txt') as f:
+    state_of_the_union = f.read()
 
 embeddings = CohereEmbeddings()
-docsearch = Chroma.from_documents(docs, embeddings, metadatas=[{"source": str(i)} for i in range(len(docs))])
+
+text_splitter = RecursiveCharacterTextSplitter(
+    chunk_size = 100,
+    chunk_overlap  = 20,
+    length_function = len,
+)
+
+texts = text_splitter.create_documents([state_of_the_union])
+
+docsearch = Chroma.from_documents(texts, embeddings)
+
+llm=Cohere(model="command-xlarge-nightly",temperature=0.7,max_tokens=1000)
+
 print(f'Done with loading and Indexing')
 
 def procdata(name):

data/Chapter-5.pdf

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:07bb2f993d156d4e111ded9faab34b814844a52a16989bbd291848046607be18
-size 771590

data/Chapter-6.pdf

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ebe4deac8a76b3657e3ceb6a8a5e996a9482084d8c4c6d318d746355aa79522e
-size 982810

data/Chapter-7.pdf

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1075e582ba6d05791d7a13ad9954e730d451c5ac0b49c4f2ca32b9cd98d1f0dc
-size 782149

data/Chapter-8.pdf

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:43237c26adf1a84b1f0a3ee5d03930d5a1703ca8a458ca5bfb2a5b8dad43a4d2
-size 1724135

data/sample.txt

@@ -1 +0,0 @@
-dummy data