Spaces:
Runtime error
Runtime error
Commit
•
88c1065
1
Parent(s):
150092a
Upload 16 files
Browse files- .gitattributes +4 -0
- 10K_Annual_Reports/Alphabet.pdf +0 -0
- 10K_Annual_Reports/Amazon.pdf +0 -0
- 10K_Annual_Reports/Apple.pdf +0 -0
- 10K_Annual_Reports/Meta.pdf +3 -0
- 10K_Annual_Reports/Microsoft.pdf +3 -0
- 10K_Annual_Reports/Netflix.pdf +3 -0
- 10K_Annual_Reports/Tesla.pdf +3 -0
- VectorStoreIndex/chroma-collections.parquet +3 -0
- VectorStoreIndex/chroma-embeddings.parquet +3 -0
- VectorStoreIndex/index/id_to_uuid_4687da76-fa8c-47cd-96a2-c9f3fc08313a.pkl +3 -0
- VectorStoreIndex/index/index_4687da76-fa8c-47cd-96a2-c9f3fc08313a.bin +3 -0
- VectorStoreIndex/index/index_metadata_4687da76-fa8c-47cd-96a2-c9f3fc08313a.pkl +3 -0
- VectorStoreIndex/index/uuid_to_id_4687da76-fa8c-47cd-96a2-c9f3fc08313a.pkl +3 -0
- app.py +46 -0
- requirements.txt +7 -0
- vectorstore.py +86 -0
.gitattributes
CHANGED
@@ -36,3 +36,7 @@ Web[[:space:]]Application/10K_Annual_Reports/Meta.pdf filter=lfs diff=lfs merge=
|
|
36 |
Web[[:space:]]Application/10K_Annual_Reports/Microsoft.pdf filter=lfs diff=lfs merge=lfs -text
|
37 |
Web[[:space:]]Application/10K_Annual_Reports/Netflix.pdf filter=lfs diff=lfs merge=lfs -text
|
38 |
Web[[:space:]]Application/10K_Annual_Reports/Tesla.pdf filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
36 |
Web[[:space:]]Application/10K_Annual_Reports/Microsoft.pdf filter=lfs diff=lfs merge=lfs -text
|
37 |
Web[[:space:]]Application/10K_Annual_Reports/Netflix.pdf filter=lfs diff=lfs merge=lfs -text
|
38 |
Web[[:space:]]Application/10K_Annual_Reports/Tesla.pdf filter=lfs diff=lfs merge=lfs -text
|
39 |
+
10K_Annual_Reports/Meta.pdf filter=lfs diff=lfs merge=lfs -text
|
40 |
+
10K_Annual_Reports/Microsoft.pdf filter=lfs diff=lfs merge=lfs -text
|
41 |
+
10K_Annual_Reports/Netflix.pdf filter=lfs diff=lfs merge=lfs -text
|
42 |
+
10K_Annual_Reports/Tesla.pdf filter=lfs diff=lfs merge=lfs -text
|
10K_Annual_Reports/Alphabet.pdf
ADDED
The diff for this file is too large to render.
See raw diff
|
|
10K_Annual_Reports/Amazon.pdf
ADDED
The diff for this file is too large to render.
See raw diff
|
|
10K_Annual_Reports/Apple.pdf
ADDED
The diff for this file is too large to render.
See raw diff
|
|
10K_Annual_Reports/Meta.pdf
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:4a2bebf058c6e947c09f9fdb510010a92f6698b458941956ad0bbdaa043ae6de
|
3 |
+
size 1111637
|
10K_Annual_Reports/Microsoft.pdf
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:50507a219c93a452c1a15e1c5bb5d01d53a97d75c1ce91ea0a9703ef7debca95
|
3 |
+
size 1547825
|
10K_Annual_Reports/Netflix.pdf
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:d95d9d4a03473863582a234e8edfd97eac97f1be9e552f9467e95dd8ce61280e
|
3 |
+
size 1410523
|
10K_Annual_Reports/Tesla.pdf
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:3a2bfbae724f9f4a7b28539993ca79c54db6ead3ff5105693a546e5a2134bbde
|
3 |
+
size 2659773
|
VectorStoreIndex/chroma-collections.parquet
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:250cf833cc54545b03d2454a5ff23eda3e047f8a3c465d29243f2e697b095848
|
3 |
+
size 557
|
VectorStoreIndex/chroma-embeddings.parquet
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:32c73aa28836865bbc6964cb3f8f0a540b9639828e39b6fa3c4ae0cb7fc7a1a3
|
3 |
+
size 114611418
|
VectorStoreIndex/index/id_to_uuid_4687da76-fa8c-47cd-96a2-c9f3fc08313a.pkl
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:c839634ab7858bf13401325e4055d5a3df0dcd5984705ecd5d83a79966363e0e
|
3 |
+
size 150307
|
VectorStoreIndex/index/index_4687da76-fa8c-47cd-96a2-c9f3fc08313a.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:4133367b79c8a0bed4a21a4885f7d35008f9bc69c9fd0b513eafcfb59faddb0b
|
3 |
+
size 29136520
|
VectorStoreIndex/index/index_metadata_4687da76-fa8c-47cd-96a2-c9f3fc08313a.pkl
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:4be5c2f38188c24bc82c3ae21db9bcbf876838e71e8b95c31435a90f960c26f2
|
3 |
+
size 74
|
VectorStoreIndex/index/uuid_to_id_4687da76-fa8c-47cd-96a2-c9f3fc08313a.pkl
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:ecdc7842716c5b96c8438096e1d1f5a276da5742cf13ea2101f83de45c0f5456
|
3 |
+
size 175727
|
app.py
ADDED
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import os, gradio
|
2 |
+
from langchain.document_loaders import UnstructuredPDFLoader
|
3 |
+
from langchain.indexes import VectorstoreIndexCreator
|
4 |
+
from vectorstore import VectorstoreIndexCreator
|
5 |
+
|
6 |
+
os.environ['OPENAI_API_KEY'] = os.getenv("OPENAI_API_KEY")
|
7 |
+
|
8 |
+
text_folder = '10K_Annual_Reports'
|
9 |
+
loaders = [UnstructuredPDFLoader(os.path.join(text_folder, fn)) for fn in os.listdir(text_folder)]
|
10 |
+
|
11 |
+
# Create the index, if it does not exist, and save it
|
12 |
+
if not os.path.isfile('VectorStoreIndex/chroma-embeddings.parquet'):
|
13 |
+
from langchain.vectorstores import Chroma
|
14 |
+
index = VectorstoreIndexCreator(vectorstore_cls=Chroma, vectorstore_kwargs={ "persist_directory": "VectorStoreIndex/"}).from_loaders(loaders)
|
15 |
+
index.vectorstore.persist()
|
16 |
+
|
17 |
+
# Load the saved index
|
18 |
+
index_saved = VectorstoreIndexCreator().from_persistent_index("VectorStoreIndex/")
|
19 |
+
|
20 |
+
description = """This is an AI conversational agent where you provide it with the annual reports of companies, and it can study it and answer any questions
|
21 |
+
you have about it. Currently, the LLM has been trained on the following companies' 10-K reports: Amazon, Apple, Alphabet (Google), Meta (Facebook), Microsoft,
|
22 |
+
Netflix and Tesla.' I plan to include more companies' 10-K reports in future.
|
23 |
+
|
24 |
+
Once the LLM is trained on a new 10-K report, it stores the vector embeddings of the document locally using ChromaDB to make the querying faster and also to
|
25 |
+
save time and money on creating the vector embeddings for the same document in future.
|
26 |
+
|
27 |
+
The LLM's universe is only the 10-K reports it has been trained on; it cannot pull information from the internet. So, you can ask it about anything that's
|
28 |
+
contained in their 10-K reports. If it cannot find an answer to your query within the 10-K reports, it will reply with "I don't know". Some example of questions
|
29 |
+
you can ask are:
|
30 |
+
|
31 |
+
- What are the risks for Tesla?
|
32 |
+
- What was Google's earnings for the last fiscal year?
|
33 |
+
- Who are the competetors of Apple?
|
34 |
+
|
35 |
+
An example of querying about something the LLM's training did not include:
|
36 |
+
|
37 |
+
- Query: "What is Tesco?"
|
38 |
+
- Response: " Tesco is not mentioned in the context, so I don't know."
|
39 |
+
"""
|
40 |
+
|
41 |
+
def chat_response(query):
|
42 |
+
return index_saved.query(query)
|
43 |
+
|
44 |
+
interface = gradio.Interface(fn=chat_response, inputs="text", outputs="text", title='Annual Reports GPT', description=description)
|
45 |
+
|
46 |
+
interface.launch() #server_name="0.0.0.0", server_port=8080, share=True)
|
requirements.txt
ADDED
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
gradio
|
2 |
+
langchain
|
3 |
+
unstructured
|
4 |
+
openai
|
5 |
+
chromadb
|
6 |
+
unstructured
|
7 |
+
tiktoken
|
vectorstore.py
ADDED
@@ -0,0 +1,86 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from typing import Any, List, Optional, Type
|
2 |
+
|
3 |
+
from pydantic import BaseModel, Extra, Field
|
4 |
+
|
5 |
+
from langchain.base_language import BaseLanguageModel
|
6 |
+
from langchain.chains.qa_with_sources.retrieval import RetrievalQAWithSourcesChain
|
7 |
+
from langchain.chains.retrieval_qa.base import RetrievalQA
|
8 |
+
from langchain.document_loaders.base import BaseLoader
|
9 |
+
from langchain.embeddings.base import Embeddings
|
10 |
+
from langchain.embeddings.openai import OpenAIEmbeddings
|
11 |
+
from langchain.llms.openai import OpenAI
|
12 |
+
from langchain.schema import Document
|
13 |
+
from langchain.text_splitter import RecursiveCharacterTextSplitter, TextSplitter
|
14 |
+
from langchain.vectorstores.base import VectorStore
|
15 |
+
from langchain.vectorstores.chroma import Chroma
|
16 |
+
|
17 |
+
|
18 |
+
def _get_default_text_splitter() -> TextSplitter:
|
19 |
+
return RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
|
20 |
+
|
21 |
+
|
22 |
+
class VectorStoreIndexWrapper(BaseModel):
|
23 |
+
"""Wrapper around a vectorstore for easy access."""
|
24 |
+
|
25 |
+
vectorstore: VectorStore
|
26 |
+
|
27 |
+
class Config:
|
28 |
+
"""Configuration for this pydantic object."""
|
29 |
+
|
30 |
+
extra = Extra.forbid
|
31 |
+
arbitrary_types_allowed = True
|
32 |
+
|
33 |
+
def query(
|
34 |
+
self, question: str, llm: Optional[BaseLanguageModel] = None, **kwargs: Any
|
35 |
+
) -> str:
|
36 |
+
"""Query the vectorstore."""
|
37 |
+
llm = llm or OpenAI(temperature=0)
|
38 |
+
chain = RetrievalQA.from_chain_type(
|
39 |
+
llm, retriever=self.vectorstore.as_retriever(), **kwargs
|
40 |
+
)
|
41 |
+
return chain.run(question)
|
42 |
+
|
43 |
+
def query_with_sources(
|
44 |
+
self, question: str, llm: Optional[BaseLanguageModel] = None, **kwargs: Any
|
45 |
+
) -> dict:
|
46 |
+
"""Query the vectorstore and get back sources."""
|
47 |
+
llm = llm or OpenAI(temperature=0)
|
48 |
+
chain = RetrievalQAWithSourcesChain.from_chain_type(
|
49 |
+
llm, retriever=self.vectorstore.as_retriever(), **kwargs
|
50 |
+
)
|
51 |
+
return chain({chain.question_key: question})
|
52 |
+
|
53 |
+
|
54 |
+
class VectorstoreIndexCreator(BaseModel):
|
55 |
+
"""Logic for creating indexes."""
|
56 |
+
|
57 |
+
vectorstore_cls: Type[VectorStore] = Chroma
|
58 |
+
embedding: Embeddings = Field(default_factory=OpenAIEmbeddings)
|
59 |
+
text_splitter: TextSplitter = Field(default_factory=_get_default_text_splitter)
|
60 |
+
vectorstore_kwargs: dict = Field(default_factory=dict)
|
61 |
+
|
62 |
+
class Config:
|
63 |
+
"""Configuration for this pydantic object."""
|
64 |
+
|
65 |
+
extra = Extra.forbid
|
66 |
+
arbitrary_types_allowed = True
|
67 |
+
|
68 |
+
def from_loaders(self, loaders: List[BaseLoader]) -> VectorStoreIndexWrapper:
|
69 |
+
"""Create a vectorstore index from loaders."""
|
70 |
+
docs = []
|
71 |
+
for loader in loaders:
|
72 |
+
docs.extend(loader.load())
|
73 |
+
return self.from_documents(docs)
|
74 |
+
|
75 |
+
def from_documents(self, documents: List[Document]) -> VectorStoreIndexWrapper:
|
76 |
+
"""Create a vectorstore index from documents."""
|
77 |
+
sub_docs = self.text_splitter.split_documents(documents)
|
78 |
+
vectorstore = self.vectorstore_cls.from_documents(
|
79 |
+
sub_docs, self.embedding, **self.vectorstore_kwargs
|
80 |
+
)
|
81 |
+
return VectorStoreIndexWrapper(vectorstore=vectorstore)
|
82 |
+
|
83 |
+
def from_persistent_index(self, path: str) -> VectorStoreIndexWrapper:
|
84 |
+
"""Load a vectorstore index from a persistent index."""
|
85 |
+
vectorstore = self.vectorstore_cls(persist_directory=path, embedding_function=self.embedding)
|
86 |
+
return VectorStoreIndexWrapper(vectorstore=vectorstore)
|