Duplicate from fengtc/langchain-pdf-openai-pinecone

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+ANSYS_Fluent_Text_Command_List.pdf filter=lfs diff=lfs merge=lfs -text

ANSYS_Fluent_Text_Command_List.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5dd5c52133e79300143e09f32410d9d0372ff27d0ef8371ea9800eb76fa5cb44
+size 4218442

README.md

@@ -0,0 +1,14 @@
+---
+title: Langchain Pdf Openai Pinecone
+emoji: ⚡
+colorFrom: indigo
+colorTo: purple
+sdk: gradio
+sdk_version: 3.39.0
+app_file: app.py
+pinned: false
+license: openrail
+duplicated_from: fengtc/langchain-pdf-openai-pinecone
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,91 @@
+from PyPDF2 import PdfReader
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.vectorstores import FAISS, Chroma, Pinecone
+from langchain.chains.question_answering import load_qa_chain
+from langchain.llms import OpenAI
+from gradio import gradio as gr
+from langchain.chat_models import ChatOpenAI
+from langchain.schema import AIMessage, HumanMessage
+
+
+import os
+# os.environ["OPENAI_API_KEY"] = 'sk-U'
+
+OPENAI_API_KEY=os.getenv('OPENAI_API_KEY')
+
+import pinecone
+
+# 初始化 pinecone
+pinecone.init(
+  api_key=os.getenv('pinecone_api_key'),
+  environment="gcp-starter"
+)
+
+index_name="pdf-index"
+
+
+pdf_files = ['./ANSYS_Fluent_Text_Command_List.pdf']
+raw_text = ''
+
+for file in pdf_files:
+    reader = PdfReader(file)
+    for i, page in enumerate(reader.pages):
+
+        text = page.extract_text()
+        if text:
+            raw_text += text
+
+
+text_splitter = CharacterTextSplitter(
+     separator="\n",
+     chunk_size=1000,
+     chunk_overlap=200,
+     length_function=len
+ )
+
+# texts = text_splitter.split_text(raw_text)
+
+with open("output.txt", "w", encoding="utf-8") as file:
+    file.write(raw_text)
+
+texts = text_splitter.split_text(raw_text)
+
+embeddings = OpenAIEmbeddings()
+
+# 持久化数据
+#docsearch = Pinecone.from_texts([t.page_content for t in texts], embeddings, index_name=index_name)
+
+# 加载数据
+docsearch = Pinecone.from_existing_index(index_name, embeddings)
+
+#embeddings = OpenAIEmbeddings()
+#embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
+#docsearch = FAISS.from_texts(texts, embeddings)
+chain = load_qa_chain(OpenAI(), chain_type="stuff")
+
+
+
+#llm = ChatOpenAI(temperature=1.0, model='gpt-3.5-turbo-0613')
+
+def predict(message, history):
+    history_langchain_format = []
+    for human, ai in history:
+        history_langchain_format.append(HumanMessage(content=human))
+        history_langchain_format.append(AIMessage(content=ai))
+    history_langchain_format.append(HumanMessage(content=message))
+    docs = docsearch.similarity_search(message)
+    response = chain.run(input_documents=docs, question=message)
+
+    partial_message = ""
+    for chunk in response:
+        if len(chunk['choices'][0]['delta']) != 0:
+            partial_message = partial_message + chunk['choices'][0]['delta']['content']
+            yield partial_message
+
+    
+    #return response
+
+gr.ChatInterface(predict).queue().launch()
+

chroma_store_chunk_size1500_two_pdf0713/chroma-collections.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cebccbc40816354875d25c5638f8e392616f13e37f8d2e066671433d2e9cfbde
+size 557

chroma_store_chunk_size1500_two_pdf0713/chroma-embeddings.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9992c6695acde8d7ef9fd5500f139930c13e53b9649b6c3a286decaf57980814
+size 37065666

chroma_store_chunk_size1500_two_pdf0713/index/id_to_uuid_ab778bed-1e82-4d31-83b7-d018b3327c38.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a9da2f18f9ffd9a55dc73c33b1debfe4ce3db4579dcccc39a15b49ee08d7362
+size 225580

chroma_store_chunk_size1500_two_pdf0713/index/index_ab778bed-1e82-4d31-83b7-d018b3327c38.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:884dcbe7d8ac8d4273dace39fa4e0e04e3c72a96265f9fb7078000d95a35fd7c
+size 22371140

chroma_store_chunk_size1500_two_pdf0713/index/index_metadata_ab778bed-1e82-4d31-83b7-d018b3327c38.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8d56612b162d70f02859a7676f698e0aa4c3feab1156d20b96a3ee3c3f067d5
+size 105

chroma_store_chunk_size1500_two_pdf0713/index/uuid_to_id_ab778bed-1e82-4d31-83b7-d018b3327c38.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a22203191ea7367c816068fab84141ae8a280166ae2172d2ec0a952a6213364
+size 263757

requirements.txt

@@ -0,0 +1,7 @@
+PyPDF2
+langchain
+openai
+docsearch
+tiktoken
+texts
+pinecone-client