Update app.py

app.py

@@ -26,46 +26,14 @@ from utils import ClaudeLLM
 from qdrant_client import models, QdrantClient
 from sentence_transformers import SentenceTransformer
 
-# embeddings = HuggingFaceEmbeddings()
-# db_art = FAISS.load_local('db_art', embeddings)
+embeddings = HuggingFaceEmbeddings()
+embeddings_1 = HuggingFaceEmbeddings(model_name = "BAAI/bge-large-en-v1.5")
+
+db_art = FAISS.load_local('db_art', embeddings)
+db_art_1 = FAISS.load_local('db_art_1', embeddings_1)
 # db_yt = FAISS.load_local('db_yt', embeddings)
 mp_docs = {}
 
-qdrant = QdrantClient(
-    "https://0a1b865d-8291-41ef-8c29-ca6c35e26391.us-east4-0.gcp.cloud.qdrant.io:6333",
-    prefer_grpc=True,
-    api_key=os.environ.get('Qdrant_Api_Key')
-)
-encoder = SentenceTransformer('BAAI/bge-large-en-v1.5')
-def q_retrieve_thoughts(query, n, db = "articles"):
-    # get collection all chunks here, to be used/ retrieved.
-    v_len = qdrant.get_collection(db).dict()['vectors_count']
-    hits = qdrant.search(
-    collection_name="articles",
-    query_vector=encoder.encode(query).tolist(),
-    limit=v_len # TO-DO: know the right number of thoughts existing maybe using get_collection
-)
-    df = pd.DataFrame.from_records([dict(hit) for hit in hits] )
-    payload = pd.DataFrame(list(df['payload'].values[:]))
-    
-    # payload.rename(columns = ['id': 'order_id'])
-    # payload['id'] = df['id']
-    
-    payload['score'] = df['score']
-    del df
-    payload.sort_values('score', ascending = False, inplace = True)
-    
-    tier_1 = payload
-    
-    chunks_1 = tier_1.groupby(['_id', ]).apply(lambda x: "\n...\n".join(x.sort_values('id')['page_content'].values)).values
-    score = tier_1.groupby(['_id', ]).apply(lambda x: x['score'].mean()).values
-    
-    tier_1_adjusted = tier_1.groupby(['_id', ]).first().reset_index()[['_id', 'title', 'url', 'author']]
-    tier_1_adjusted['content'] = list(chunks_1)
-    tier_1_adjusted['score'] = score
-    tier_1_adjusted = tier_1_adjusted[tier_1_adjusted['score']>0.5]
-    tier_1_adjusted.sort_values('score', ascending = False, inplace = True)
-    return {'tier 1':tier_1_adjusted, }
 
 def retrieve_thoughts(query, n, db):
 
@@ -120,6 +88,27 @@ def qa_retrieve_art(query,):
     return {'Reference': reference}
 
 
+def qa_retrieve_bge(query,):
+
+    docs = ""
+
+    global db_art_1
+
+    global mp_docs
+    thoughts = retrieve_thoughts(query, 0, db_art_1)
+    if not(thoughts):
+
+        if mp_docs:
+            thoughts = mp_docs
+    else:
+        mp_docs = thoughts
+
+    tier_1 = thoughts['tier 1']
+    
+    reference = tier_1[['_id', 'url', 'author', 'title', 'chunks', 'score']].to_dict('records')
+
+    return {'Reference': reference}
+
 def qa_retrieve_yt(query,):
 
     docs = ""
@@ -145,13 +134,15 @@ def flush():
   return None
 
 
-
-ref_art = gr.Interface(fn=qa_retrieve_art, label="Articles",
+ref_art_1 = gr.Interface(fn=qa_retrieve_art, label="bge Articles",
+                     inputs=gr.inputs.Textbox(lines=5, label="what would you like to learn about?"),
+                     outputs = gr.components.JSON(label="articles"))
+ref_art = gr.Interface(fn=qa_retrieve_art, label="mpnet Articles",
                      inputs=gr.inputs.Textbox(lines=5, label="what would you like to learn about?"),
                      outputs = gr.components.JSON(label="articles"))
 # ref_yt = gr.Interface(fn=qa_retrieve_yt, label="Youtube",
 #                      inputs=gr.inputs.Textbox(lines=5, label="what would you like to learn about?"),
 #                      outputs = gr.components.JSON(label="youtube"),title = "youtube", examples=examples)
-demo = gr.Parallel( ref_art,)
+demo = gr.Parallel( ref_art_1, ref_art_1)
 
 demo.launch()