I

pages/Tesla_Alerts.py

@@ -1,21 +1,20 @@
 from langchain.prompts import PromptTemplate
 from langchain.output_parsers import PydanticOutputParser
+from langchain.chat_models import ChatOpenAI
 
-from llama_index import VectorStoreIndex, ServiceContext, StorageContext
+from llama_index import VectorStoreIndex, ServiceContext, StorageContext,  download_loader, SimpleDirectoryReader
 from llama_index.vector_stores import FaissVectorStore
 from llama_index.tools import QueryEngineTool, ToolMetadata
 from llama_index.query_engine import SubQuestionQueryEngine
 from llama_index.embeddings import OpenAIEmbedding
 from llama_index.schema import Document
 from llama_index.node_parser import UnstructuredElementNodeParser
-
-from src.utils import get_model, process_pdf2
+from llama_index.llms import OpenAI
 
 import streamlit as st
 import os
 import faiss
 import time
-from pypdf import PdfReader
 
 
 st.set_page_config(page_title="Yield Case Analyzer", page_icon=":card_index_dividers:", initial_sidebar_state="expanded", layout="wide")
@@ -25,24 +24,15 @@ st.info("""
 Begin by uploading the case report in PDF format. Afterward, click on 'Process Document'. Once the document has been processed. You can enter question and click send, system will answer your question.
 """)
 
-def process_pdf(pdf):
-    file = PdfReader(pdf)
-    print("in process pdf")
-    document_list = []
-    for page in file.pages:
-        document_list.append(Document(text=str(page.extract_text())))
-    print("in process pdf 1")
-
-    node_paser = UnstructuredElementNodeParser()
-    print("in process pdf 1")
-
-    nodes = node_paser.get_nodes_from_documents(document_list, show_progress=True)
 
-    return nodes
+def get_model(model_name):
+    OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
+    if model_name == "openai":
+        model = ChatOpenAI(openai_api_key=OPENAI_API_KEY, model_name="gpt-3.5-turbo")
+    return model
 
-
-def get_vector_index(nodes, vector_store):
-    print(nodes)
+def get_vector_index(docs, vector_store):
+    print(docs)
     llm = get_model("openai")
     if vector_store == "faiss":
         d = 1536
@@ -52,12 +42,12 @@ def get_vector_index(nodes, vector_store):
         # embed_model = OpenAIEmbedding()
         # service_context = ServiceContext.from_defaults(embed_model=embed_model)
         service_context = ServiceContext.from_defaults(llm=llm)
-        index = VectorStoreIndex(nodes,
+        index = VectorStoreIndex(docs,
             service_context=service_context,
             storage_context=storage_context
         )
     elif vector_store == "simple":
-        index = VectorStoreIndex.from_documents(nodes)
+        index = VectorStoreIndex.from_documents(docs)
 
 
     return index
@@ -89,7 +79,7 @@ def get_query_engine(engine):
             query_engine=engine,
             metadata=ToolMetadata(
                 name="Alert Report",
-                description=f"Provides information about the cases from its case report.",
+                description=f"Provides information about the alerts from alerts files uploaded.",
             ),
         ),
     ]
@@ -111,20 +101,39 @@ os.environ["OPENAI_API_KEY"] = OPENAI_API_KEY
 
 
 if OPENAI_API_KEY:
-    pptx_files = st.sidebar.file_uploader("Upload the case report in PDF format", type="pptx")
+    files_uploaded = st.sidebar.file_uploader("Upload the case report in PDF format", type="pptx")
     st.sidebar.info("""
     Example pdf reports you can upload here:
     """)
 
     if st.sidebar.button("Process Document"):
         with st.spinner("Processing Document..."):
-            nodes = process_pptx(pptx_files)
-            st.session_state.index = get_vector_index(nodes, vector_store="faiss")
-            #st.session_state.index = get_vector_index(nodes, vector_store="simple")
-            st.session_state.process_doc = True
+
+        data_dir = "./data"
+        if not os.path.exists(data_dir):
+            os.makedirs(data_dir)
+
+        for file in files_uploaded:
+            print(f'file named {file.name}')
+            fname=f'{data_dir}/{file.name}'
+            with open(fname, 'wb') as f:
+                f.write(file.read())
+
+        def fmetadata(dummy: str): return {"file_path": ""}
+
+        PptxReader = download_loader("PptxReader")
+        loader =  SimpleDirectoryReader(input_dir=data_dir, file_extractor={".pptx": PptxReader(),}, file_metadata=fmetadata)
+
+        documents = loader.load_data()
+        for doc in documents:
+            doc.metadata["file_path"]=""
+
+        st.session_state.index = get_vector_index(documents, vector_store="faiss")
+        #st.session_state.index = get_vector_index(documents, vector_store="simple")
+        st.session_state.process_doc = True
         st.toast("Document Processsed!")
 
-    st.session_state.process_doc = True
+    #st.session_state.process_doc = True
 
     if st.session_state.process_doc:
         search_text = st.text_input("Enter your question")
@@ -134,10 +143,11 @@ if OPENAI_API_KEY:
 
             st.write("Alert search result...")
             response = generate_insight(engine, search_text)
-            st.session_state["end_time"] = "{:.2f}".format((time.time() - start_time))
+            st.write(response)
+            #st.session_state["end_time"] = "{:.2f}".format((time.time() - start_time))
 
             st.toast("Report Analysis Complete!")
 
-        if st.session_state.end_time:
-            st.write("Report Analysis Time: ", st.session_state.end_time, "s")
+        #if st.session_state.end_time:
+        #    st.write("Report Analysis Time: ", st.session_state.end_time, "s")
 

requirements.txt

@@ -1,8 +1,10 @@
 llama-index
 llama_hub
+langchain
 transformers
 accelerate
 openai
 pypdf
 streamlit
-chromadb
+chromadb
+faiss