Manglik-R commited on
Commit
d522487
1 Parent(s): 6eb7365

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +6 -8
app.py CHANGED
@@ -1,6 +1,6 @@
1
  import gradio as gr
2
  from langchain.document_loaders import OnlinePDFLoader
3
- from langchain.text_splitter import RecursiveCharacterTextSplitter
4
  from langchain.embeddings import HuggingFaceHubEmbeddings
5
  from langchain.vectorstores import FAISS
6
  from langchain.llms import HuggingFaceHub
@@ -21,16 +21,14 @@ def pdf_changes(pdf_doc):
21
 
22
  loader = OnlinePDFLoader(pdf_doc.name)
23
  pages = loader.load_and_split()
24
- text_splitter = RecursiveCharacterTextSplitter(
25
- chunk_size=256,
26
- chunk_overlap=8,
27
- separators=['\n\n', '\n', '(?=>\. )', ' ', '']
28
  )
29
  docs = text_splitter.split_documents(pages)
30
  embeddings = HuggingFaceHubEmbeddings()
31
  db = FAISS.from_documents(docs, embeddings)
32
-
33
- llm=HuggingFaceHub(repo_id="google/flan-t5-xxl", model_kwargs={"temperature":0.1, "max_new_tokens":250})
34
  global qa
35
  qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=db.as_retriever())
36
  return "Ready"
@@ -39,7 +37,7 @@ def book_changes(book):
39
  db = FAISS.load_local( book , embeddings = HuggingFaceHubEmbeddings() )
40
  llm=HuggingFaceHub(repo_id="google/flan-t5-xxl", model_kwargs={"temperature":0.1, "max_new_tokens":250})
41
  global qa
42
- qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=db.as_retriever())
43
  return "Ready"
44
 
45
 
 
1
  import gradio as gr
2
  from langchain.document_loaders import OnlinePDFLoader
3
+ from langchain.text_splitter import CharacterTextSplitter
4
  from langchain.embeddings import HuggingFaceHubEmbeddings
5
  from langchain.vectorstores import FAISS
6
  from langchain.llms import HuggingFaceHub
 
21
 
22
  loader = OnlinePDFLoader(pdf_doc.name)
23
  pages = loader.load_and_split()
24
+ text_splitter = CharacterTextSplitter(
25
+ chunk_size=350,
26
+ chunk_overlap=0,
 
27
  )
28
  docs = text_splitter.split_documents(pages)
29
  embeddings = HuggingFaceHubEmbeddings()
30
  db = FAISS.from_documents(docs, embeddings)
31
+ llm = HuggingFaceHub(repo_id="google/flan-ul2", model_kwargs={"temperature":0.1, "max_new_tokens":300})
 
32
  global qa
33
  qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=db.as_retriever())
34
  return "Ready"
 
37
  db = FAISS.load_local( book , embeddings = HuggingFaceHubEmbeddings() )
38
  llm=HuggingFaceHub(repo_id="google/flan-t5-xxl", model_kwargs={"temperature":0.1, "max_new_tokens":250})
39
  global qa
40
+ qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=db.as_retriever(), return_source_documents=True)
41
  return "Ready"
42
 
43