""" Page for similarities """ ################ # DEPENDENCIES # ################ import streamlit as st import pandas as pd from scipy.sparse import load_npz import pickle from sentence_transformers import SentenceTransformer from modules.multimatch_result_table import show_multi_table from modules.singlematch_result_table import show_single_table from functions.filter_projects import filter_projects from functions.calc_matches import calc_matches from functions.same_country_filter import same_country_filter from functions.single_similar import find_similar import psutil import os import gc def get_process_memory(): process = psutil.Process(os.getpid()) return process.memory_info().rss / (1024 * 1024) # Catch DATA # Load Similarity matrix @st.cache_data def load_sim_matrix(): loaded_matrix = load_npz("src/extended_similarities.npz") dense_matrix = loaded_matrix.toarray() return dense_matrix # Load Non Similar Orga Matrix @st.cache_data def load_nonsameorga_sim_matrix(): loaded_matrix = load_npz("src/extended_similarities_nonsimorga.npz") dense_matrix = loaded_matrix.toarray() return dense_matrix # Load Projects DFs @st.cache_data def load_projects(): orgas_df = pd.read_csv("src/projects/project_orgas.csv") region_df = pd.read_csv("src/projects/project_region.csv") sector_df = pd.read_csv("src/projects/project_sector.csv") status_df = pd.read_csv("src/projects/project_status.csv") texts_df = pd.read_csv("src/projects/project_texts.csv") projects_df = pd.merge(orgas_df, region_df, on='iati_id', how='inner') projects_df = pd.merge(projects_df, sector_df, on='iati_id', how='inner') projects_df = pd.merge(projects_df, status_df, on='iati_id', how='inner') projects_df = pd.merge(projects_df, texts_df, on='iati_id', how='inner') iati_search_list = [f'{row.iati_id}' for row in projects_df.itertuples()] title_search_list = [f'{row.title_main} ({row.orga_abbreviation.upper()})' for row in projects_df.itertuples()] return projects_df, iati_search_list, title_search_list # Load CRS 3 data @st.cache_data def getCRS3(): # Read in CRS3 CODELISTS crs3_df = pd.read_csv('src/codelists/crs3_codes.csv') CRS3_CODES = crs3_df['code'].tolist() CRS3_NAME = crs3_df['name'].tolist() CRS3_MERGED = {f"{name} - {code}": code for name, code in zip(CRS3_NAME, CRS3_CODES)} return CRS3_MERGED # Load CRS 5 data @st.cache_data def getCRS5(): # Read in CRS3 CODELISTS crs5_df = pd.read_csv('src/codelists/crs5_codes.csv') CRS5_CODES = crs5_df['code'].tolist() CRS5_NAME = crs5_df['name'].tolist() CRS5_MERGED = {code: [f"{name} - {code}"] for name, code in zip(CRS5_NAME, CRS5_CODES)} return CRS5_MERGED # Load SDG data @st.cache_data def getSDG(): # Read in SDG CODELISTS sdg_df = pd.read_csv('src/codelists/sdg_goals.csv') SDG_NAMES = sdg_df['name'].tolist() return SDG_NAMES # Load Country Data @st.cache_data def getCountry(): # Read in countries from codelist country_df = pd.read_csv('src/codelists/country_codes_ISO3166-1alpha-2.csv') COUNTRY_CODES = country_df['Alpha-2 code'].tolist() COUNTRY_NAMES = country_df['Country'].tolist() COUNTRY_OPTION_LIST = [f"{COUNTRY_NAMES[i]} ({COUNTRY_CODES[i][-3:-1].upper()})"for i in range(len(COUNTRY_NAMES))] return COUNTRY_OPTION_LIST # Load Sentence Transformer Model @st.cache_resource def load_model(): model = SentenceTransformer('all-MiniLM-L6-v2') return model # Load Embeddings @st.cache_data def load_embeddings_and_index(): # Load embeddings with open("src/embeddings.pkl", "rb") as fIn: stored_data = pickle.load(fIn) embeddings = stored_data["embeddings"] return embeddings # USE CACHE FUNCTIONS sim_matrix = load_sim_matrix() nonsameorgas_sim_matrix = load_nonsameorga_sim_matrix() projects_df, iati_search_list, title_search_list = load_projects() CRS3_MERGED = getCRS3() CRS5_MERGED = getCRS5() SDG_NAMES = getSDG() COUNTRY_OPTION_LIST = getCountry() # LOAD MODEL FROM CACHE FO SEMANTIC SEARCH model = load_model() embeddings = load_embeddings_and_index() def show_multi_matching_page(): #st.write(f"Current RAM usage of this app: {get_process_memory():.2f} MB") st.caption(""" Multi-project matching allows you to search for projects within the selected filter setting to find another similar project. The second matching project must not qualify for the selected filter and is calculated by an AI similarity score comparing the projects. This App includes all in the IATI Database listed Projects of following Organizations: BMZ, KFW, GIZ, IAD, ADB, AFDB, EIB, WB, WBTF and the German Federal Foreign Office (AA). """) col1, col2, col3 = st.columns([10, 1, 10]) with col1: st.subheader("Sector Filter (required)") st.caption(""" Sector filters must be applied to see results. The CRS5 and CRS3 classifications organise development aid into categories, with the 5-digit level providing more specific detail within the broader 3-digit categories. The SDGs are 17 UN goals that aim to achieve global sustainability, peace and prosperity by 2030. Futhermore you can Search for projects with the query field. """) with col3: st.subheader("Additional Filters") st.caption(""" The additional filters allow for a more detailed search for the Multi-Project Matching. """) st.session_state.crs5_option_disabled = True col1, col2, col3 = st.columns([10, 1, 10]) with col1: # CRS 3 SELECTION crs3_option = st.multiselect( 'CRS 3', CRS3_MERGED, placeholder="Select a CRS 3 code" ) # CRS 5 SELECTION ## Only enable crs5 select field when crs3 code is selected if crs3_option != []: st.session_state.crs5_option_disabled = False ## define list of crs5 codes dependend on crs3 codes crs5_list = [txt[0].replace('"', "") for crs3_item in crs3_option for code, txt in CRS5_MERGED.items() if str(code)[:3] == str(crs3_item)[-3:]] ## crs5 select field crs5_option = st.multiselect( 'CRS 5', crs5_list, placeholder="Select a CRS 5 code", disabled=st.session_state.crs5_option_disabled ) # SDG SELECTION sdg_option = st.selectbox( label = 'Sustainable Development Goal (SDG)', index = None, placeholder = "Select a SDG", options = SDG_NAMES[:-1], ) # SEARCH BOX query = st.text_input("Search Query") with col3: # COUNTRY SELECTION country_option = st.multiselect( 'Country / Countries', COUNTRY_OPTION_LIST, placeholder="All countries selected" ) # ORGA SELECTION orga_abbreviation = projects_df["orga_abbreviation"].unique() orga_full_names = projects_df["orga_full_name"].unique() orga_list = [f"{orga_full_names[i]} ({orga_abbreviation[i].upper()})"for i in range(len(orga_abbreviation))] orga_option = st.multiselect( 'Development Bank / Organization', orga_list, placeholder="All organizations selected" ) different_orga_checkbox = st.checkbox("Only matches between different organizations", value=True) filterd_country_only_checkbox = st.checkbox("Only matches between filtered countries", value=True) # CRS CODE LIST crs3_list = [i[-3:] for i in crs3_option] crs5_list = [i[-5:] for i in crs5_option] # SDG CODE LIST if sdg_option != None: sdg_str = sdg_option.split(".")[0] print(sdg_str) else: sdg_str = "" # COUNTRY CODES LIST country_code_list = [option[-3:-1] for option in country_option] # ORGANIZATION CODES LIST orga_code_list = [option.split("(")[1][:-1].lower() for option in orga_option] # FILTER DF WITH SELECTED FILTER OPTIONS TOP_X_PROJECTS = 30 filtered_df = filter_projects(projects_df, crs3_list, crs5_list, sdg_str, country_code_list, orga_code_list, query, model, embeddings, TOP_X_PROJECTS) if isinstance(filtered_df, pd.DataFrame) and len(filtered_df) != 0: # FIND MATCHES ## If only same country checkbox i sactivated if filterd_country_only_checkbox: compare_df = same_country_filter(projects_df, country_code_list) else: compare_df = projects_df ## if show only different orgas checkbox is activated if different_orga_checkbox: p1_df, p2_df = calc_matches(filtered_df, compare_df, nonsameorgas_sim_matrix, TOP_X_PROJECTS) else: p1_df, p2_df = calc_matches(filtered_df, compare_df, sim_matrix, TOP_X_PROJECTS) # SHOW THE RESULT show_multi_table(p1_df, p2_df) del p1_df, p2_df else: st.write("-----") col1, col2, col3 = st.columns([1.5, 1, 1.5]) with col2: st.write(" ") st.markdown("There are no results for the applied filter. Try another filter!") del crs3_list, crs5_list, sdg_str, filtered_df gc.collect() def show_single_matching_page(): search_option = st.selectbox( label = 'Select how you want to search', index = 0, placeholder = " ", options = ["Search with IATI ID", "Search with project title"], ) if search_option == "Search with IATI ID": search_list = iati_search_list else: search_list = title_search_list project_option = st.selectbox( label = 'Search for a project with the IATI ID or teh project title', index = None, placeholder = " ", options = search_list, ) #different_orga_checkbox_ = st.checkbox("Only matches for organisations other than the selected project") #filterd_country_only_checkbox_ = st.checkbox("Only matches in the same countries as project selection") #selected_index = None if project_option: selected_index = search_list.index(project_option) """ if filterd_country_only_checkbox_: country = projects_df.iloc[selected_index]["country"] same_country_df = projects_df[projects_df['country'] == country] else: same_country_df = projects_df if different_orga_checkbox_: orga = projects_df.iloc[selected_index]["orga_abbreviation"] different_orga_df = same_country_df[same_country_df['orga_abbreviation'] != orga] else: different_orga_df = same_country_df """ top_projects_df = find_similar(selected_index, sim_matrix, projects_df, 10) show_single_table(top_projects_df)