Update app.py

app.py

@@ -66,6 +66,8 @@ def get_chunks():
                            'language', 'start_year', 'end_year','poli_trager'], inplace=True)
     giz_df.rename(columns = {'project_name':'title_main','countries':'country_name',
                             'client':'org','project_description':'description_main'}, inplace=True)
+    giz_df['source'] = 'GIZ_WORLDWIDE'
+    giz_df['status'] = "None"
     df = pd.concat([projects_df,giz_df],ignore_index=True)
     print(df.columns)
 
@@ -75,12 +77,9 @@ def get_chunks():
     placeholder= []
     for i in range(len(giz_df)):
         placeholder.append(Document(page_content= giz_df.loc[i,'chunks'], 
-                                metadata={"iati_id": giz_df.loc[i,'iati_id'],
-                                        "iati_orga_id":giz_df.loc[i,'iati_orga_id'],
+                                metadata={"id": giz_df.loc[i,'id'],
+                                        "org":giz_df.loc[i,'org'],
                                         "country_name":str(giz_df.loc[i,'country_name']),
-                                        "crs_5_name": giz_df.loc[i,'crs_5_name'],
-                                        "crs_3_name": giz_df.loc[i,'crs_3_name'],
-                                        "sgd_pred_str":giz_df.loc[i,'sgd_pred_str'],
                                         "status":giz_df.loc[i,'status'],
                                         "title_main":giz_df.loc[i,'title_main'],}))
     return placeholder
@@ -112,17 +111,15 @@ def embed_chunks(chunks):
         encode_kwargs = {'normalize_embeddings': True},
         model_name='BAAI/bge-m3'
     )
-    sparse_embeddings = FastEmbedSparse(model_name="Qdrant/bm25")
+    #sparse_embeddings = FastEmbedSparse(model_name="Qdrant/bm25")
     # placeholder for collection
     print("starting embedding")
     qdrant_collections = {}
-    qdrant_collections['iati'] = Qdrant.from_documents(
+    qdrant_collections['all'] = Qdrant.from_documents(
                 chunks,
                 embeddings,
-                sparse_embeddings = sparse_embeddings,
                 path="/data/local_qdrant",
-                collection_name='iati',
-                retrieval_mode=RetrievalMode.HYBRID,
+                collection_name='all',
             )
             
     print(qdrant_collections)