pinecone fixes

app_pinecode.py

@@ -58,7 +58,7 @@ if __name__ == "__main__":
         # Every form must have a submit button.
         submitted = st.form_submit_button("Buscar")
     
-    results = search("que es el adres", retriever_top_k=5, reader_top_k=3)
+    #results = search("que es el adres", retriever_top_k=5, reader_top_k=3)
 
     # on submit we execute search
     if(submitted):

config.py

@@ -2,7 +2,7 @@ class Config():
     es_host = "saimon-askwdemocracy.es.us-central1.gcp.cloud.es.io"
     es_user = "elastic"
     es_password = "53f2a7a9-ea9d-4fd2-a8bc-f471b67f0262"
-    proposals_index = "semantic-text-search"
+    proposals_index = "docsreloaded"
     reader_model_name_or_path = "deepset/xlm-roberta-base-squad2-distilled"
     #reader_model_name_or_path = "deepset/xlm-roberta-base-squad2"
     use_gpu = True

pinecode_quieries.py

@@ -46,6 +46,9 @@ class PineconeRetriever(BaseComponent):
         self.index = pinecone.Index(index_name)
 
     def run(self, query: str, top_k: Optional[int]):
+        #TODO: problema de versiones del cliente de pinecone. en esta versión retorna una coleción de resutsl con los matches.
+        #TODO2: no esta el contenido dentro de los metadatos, ni en ningun lado. normal? , workarround. guararlo en otrometado.
+        #TODO3: hacer consulta por id, pastelarlo desde el colab que si retorna?
         # process the inputs
         vector_embeddings = self.sts_model.encode(query).tolist()
         response = self.index.query([vector_embeddings], top_k=top_k, include_metadata=True)
@@ -57,7 +60,7 @@ class PineconeRetriever(BaseComponent):
                       'source': d["metadata"]['source']
                       }
                 )
-                for d in response["matches"]
+                for d in response['matches']
         ]
         output = {"documents": docs, "query": query}
         return output, "output_1"
@@ -84,36 +87,37 @@ class PinecodeProposalQueries(DocumentQueries):
             self.reader = reader
         
         #pinecone.init(api_key=es_password, environment="us-east1-gcp")
-        index_name = "semantic-text-search"
+        index_name = es_index
         
         self.document_store = PineconeDocumentStore(
             api_key=es_password,
             environment = "us-east1-gcp",
             index=index_name,
             similarity="cosine",
-            embedding_dim=384
+            embedding_dim=384,
+            metadata_config = {"indexed": ["title", "content"]}
         )
         self.pipe = Pipeline()
         pinecone_retriever = PineconeRetriever("sentence-transformers/multi-qa-MiniLM-L6-cos-v1", 
                                                es_password, "us-east1-gcp", 
                                                index_name)
-        self.pipe.add_node(component=pinecone_retriever, name="Retriever", inputs=["Query"])
-        self.pipe.add_node(component=self.reader, name="Reader", inputs=["Retriever"])
+        #self.pipe.add_node(component=pinecone_retriever, name="Retriever", inputs=["Query"])
+        #self.pipe.add_node(component=self.reader, name="Reader", inputs=["Retriever"])
                
         # #self.retriever = BM25Retriever(document_store = self.document_store)
-        # self.retriever = EmbeddingRetriever(
-        #     document_store=self.document_store,
-        #     #embedding_model="multi-qa-distilbert-dot-v1",
-        #     embedding_model = "sentence-transformers/msmarco-MiniLM-L6-cos-v5",
-        #     model_format="sentence_transformers"
-        # )
+        self.retriever = EmbeddingRetriever(
+            document_store=self.document_store,
+            #embedding_model="multi-qa-distilbert-dot-v1",
+            embedding_model = "sentence-transformers/multi-qa-MiniLM-L6-cos-v1",
+            model_format="sentence_transformers"
+        )
         
         # retriever_model = "sentence-transformers/multi-qa-mpnet-base-dot-v1"
 
 
         
         #self.document_store.update_embeddings(self.retriever, update_existing_embeddings=False)
-        #self.pipe = ExtractiveQAPipeline (reader = self.reader, retriever = self.retriever)
+        self.pipe = ExtractiveQAPipeline (reader = self.reader, retriever = self.retriever)
         #self.pipe = DocumentSearchPipeline(self.retriever)
 
     def search_by_query(self, query : str, retriever_top_k: int, reader_top_k: int, es_index: str = None) :

requirements.txt

@@ -1,4 +1,4 @@
-transformers==4.19.2
-torch==1.10.2
-#farm-haystack==1.5.0
-farm-haystack[pinecone]==1.5.0
+sentence-transformers
+farm-haystack[pinecone]
+pinecone-client
+streamlit