djangomango commited on
Commit
4ea5541
1 Parent(s): 6f5c8bd

Upload 14 files

Browse files
Search-Your-PDF-App-main/.gitignore ADDED
@@ -0,0 +1,160 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .nox/
43
+ .coverage
44
+ .coverage.*
45
+ .cache
46
+ nosetests.xml
47
+ coverage.xml
48
+ *.cover
49
+ *.py,cover
50
+ .hypothesis/
51
+ .pytest_cache/
52
+ cover/
53
+
54
+ # Translations
55
+ *.mo
56
+ *.pot
57
+
58
+ # Django stuff:
59
+ *.log
60
+ local_settings.py
61
+ db.sqlite3
62
+ db.sqlite3-journal
63
+
64
+ # Flask stuff:
65
+ instance/
66
+ .webassets-cache
67
+
68
+ # Scrapy stuff:
69
+ .scrapy
70
+
71
+ # Sphinx documentation
72
+ docs/_build/
73
+
74
+ # PyBuilder
75
+ .pybuilder/
76
+ target/
77
+
78
+ # Jupyter Notebook
79
+ .ipynb_checkpoints
80
+
81
+ # IPython
82
+ profile_default/
83
+ ipython_config.py
84
+
85
+ # pyenv
86
+ # For a library or package, you might want to ignore these files since the code is
87
+ # intended to run in multiple environments; otherwise, check them in:
88
+ # .python-version
89
+
90
+ # pipenv
91
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
93
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
94
+ # install all needed dependencies.
95
+ #Pipfile.lock
96
+
97
+ # poetry
98
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
99
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
100
+ # commonly ignored for libraries.
101
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
102
+ #poetry.lock
103
+
104
+ # pdm
105
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
106
+ #pdm.lock
107
+ # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
108
+ # in version control.
109
+ # https://pdm.fming.dev/#use-with-ide
110
+ .pdm.toml
111
+
112
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
113
+ __pypackages__/
114
+
115
+ # Celery stuff
116
+ celerybeat-schedule
117
+ celerybeat.pid
118
+
119
+ # SageMath parsed files
120
+ *.sage.py
121
+
122
+ # Environments
123
+ .env
124
+ .venv
125
+ env/
126
+ venv/
127
+ ENV/
128
+ env.bak/
129
+ venv.bak/
130
+
131
+ # Spyder project settings
132
+ .spyderproject
133
+ .spyproject
134
+
135
+ # Rope project settings
136
+ .ropeproject
137
+
138
+ # mkdocs documentation
139
+ /site
140
+
141
+ # mypy
142
+ .mypy_cache/
143
+ .dmypy.json
144
+ dmypy.json
145
+
146
+ # Pyre type checker
147
+ .pyre/
148
+
149
+ # pytype static type analyzer
150
+ .pytype/
151
+
152
+ # Cython debug symbols
153
+ cython_debug/
154
+
155
+ # PyCharm
156
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
157
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
158
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
159
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
160
+ #.idea/
Search-Your-PDF-App-main/LICENSE ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ MIT License
2
+
3
+ Copyright (c) 2023 AI Anytime
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
Search-Your-PDF-App-main/README.md ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ # Search-Your-PDF-App
2
+ Search Your PDF App using Langchain, ChromaDB, Sentence Transformers, and LaMiNi LM Model. This app is completely powered by Open Source Models. No OpenAI key is required.
Search-Your-PDF-App-main/app.py ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+ from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
3
+ from transformers import pipeline
4
+ import torch
5
+ import base64
6
+ import textwrap
7
+ from langchain.embeddings import SentenceTransformerEmbeddings
8
+ from langchain.vectorstores import Chroma
9
+ from langchain.llms import HuggingFacePipeline
10
+ from langchain.chains import RetrievalQA
11
+ from constants import CHROMA_SETTINGS
12
+
13
+ #model and tokenizer loading
14
+ checkpoint = "LaMini-T5-738M"
15
+ tokenizer = AutoTokenizer.from_pretrained(checkpoint)
16
+ base_model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint, device_map='auto', torch_dtype=torch.float32)
17
+
18
+ @st.cache_resource
19
+ def llm_pipeline():
20
+ pipe = pipeline(
21
+ 'text2text-generation',
22
+ model = base_model,
23
+ tokenizer = tokenizer,
24
+ max_length = 256,
25
+ do_sample=True,
26
+ temperature = 0.3,
27
+ top_p = 0.95
28
+ )
29
+ local_llm = HuggingFacePipeline(pipeline=pipe)
30
+ return local_llm
31
+
32
+ @st.cache_resource
33
+ def qa_llm():
34
+ llm = llm_pipeline()
35
+ embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
36
+ db = Chroma(persist_directory="db", embedding_function=embeddings, client_settings=CHROMA_SETTINGS)
37
+ retriever = db.as_retriever()
38
+ qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=retriever, return_source_documents=True)
39
+ return qa
40
+
41
+ def process_answer(instruction):
42
+ response = ''
43
+ instruction = instruction
44
+ qa = qa_llm()
45
+ generated_text = qa(instruction)
46
+ answer = generated_text['result']
47
+ # metadata = generated_text['metadata']
48
+ # for text in generated_text:
49
+
50
+ # print(answer)
51
+
52
+ # wrapped_text = textwrap.fill(response, 100)
53
+ # return wrapped_text
54
+ return answer,generated_text
55
+
56
+ def main():
57
+ st.title("Search Your PDF 🐦📄")
58
+ with st.expander("About the App"):
59
+ st.markdown(
60
+ """
61
+ This is a Generative AI powered Question and Answering app that responds to questions about your PDF File.
62
+ """
63
+ )
64
+ question = st.text_area("Enter your Question")
65
+ if st.button("Ask"):
66
+ st.info("Your Question: " + question)
67
+
68
+ st.info("Your Answer")
69
+ answer, metadata = process_answer(question)
70
+ st.write(answer)
71
+ st.write(metadata)
72
+
73
+
74
+ if __name__ == '__main__':
75
+ main()
76
+
Search-Your-PDF-App-main/constants.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from chromadb.config import Settings
3
+
4
+ #Define the chroma settings
5
+ CHROMA_SETTINGS = Settings(
6
+ chroma_db_impl = 'duckdb+parquet',
7
+ persist_directory = "db",
8
+ anonymized_telemetry = False
9
+ )
Search-Your-PDF-App-main/db/chroma-collections.parquet ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f50ad2ffaf7bcd328753a4333a488dc2f354d5c5b664ed15b7e61093c3b45676
3
+ size 557
Search-Your-PDF-App-main/db/chroma-embeddings.parquet ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9f44cb845a13ba159ff3d4423fcf8f8dc37293a625855e1a7a74ce27adc4972f
3
+ size 37262
Search-Your-PDF-App-main/db/index/id_to_uuid_0ff18d02-2c2d-48fc-a6a8-c763775f8664.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:91b1e584dd670e5cf77dcfb49222fb0440bbdef5458f3586b2e197c7bcf82fb0
3
+ size 444
Search-Your-PDF-App-main/db/index/index_0ff18d02-2c2d-48fc-a6a8-c763775f8664.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8bb328603e4c6019ba5abfd6d1f81679e2f9cb9bf912eb2e34e6022a8391f8e9
3
+ size 21936
Search-Your-PDF-App-main/db/index/index_metadata_0ff18d02-2c2d-48fc-a6a8-c763775f8664.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:765ce47a48c59b5d5ef9d722a22d88f820ca730b9508407c91a311af65fa9789
3
+ size 103
Search-Your-PDF-App-main/db/index/uuid_to_id_0ff18d02-2c2d-48fc-a6a8-c763775f8664.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1fa55736b4535834632bb88f75ce4f0773149729cadfbf42b5a7edc4270d1918
3
+ size 497
Search-Your-PDF-App-main/docs/fastfacts-what-is-climate-change.pdf ADDED
Binary file (390 kB). View file
 
Search-Your-PDF-App-main/ingest.py ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from langchain.text_splitter import RecursiveCharacterTextSplitter
2
+ from langchain.document_loaders import PyPDFLoader, DirectoryLoader, PDFMinerLoader
3
+ from langchain.embeddings import SentenceTransformerEmbeddings
4
+ from langchain.vectorstores import Chroma
5
+ import os
6
+ from constants import CHROMA_SETTINGS
7
+
8
+
9
+ persist_directory = "db"
10
+
11
+
12
+ def main():
13
+ for root, dirs, files in os.walk("docs"):
14
+ for file in files:
15
+ if file.endswith(".pdf"):
16
+ print(file)
17
+ loader = PDFMinerLoader(os.path.join(root, file))
18
+ documents = loader.load()
19
+ text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
20
+ texts = text_splitter.split_documents(documents)
21
+ #create embeddings here
22
+ embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
23
+ db = Chroma.from_documents(texts, embeddings, persist_directory=persist_directory, client_settings=CHROMA_SETTINGS)
24
+ db.persist()
25
+ db=None
26
+
27
+ if __name__ == "__main__":
28
+ main()
Search-Your-PDF-App-main/requirements.txt ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ langchain
2
+ streamlit
3
+ transformers
4
+ requests
5
+ torch
6
+ einops
7
+ accelerate
8
+ bitsandbytes
9
+ pdfminer.six
10
+ bs4
11
+ sentence_transformers
12
+ chromadb