functions/calc_matches.py

import pandas as pd
import numpy as np
from scipy.sparse import csr_matrix, lil_matrix

"""
def calc_matches(filtered_df, project_df, similarity_matrix, top_x):
    # matching project2 can be nay project
    # indecies (rows) = project1
    # columns = project2
    # -> find matches

    # filter out all row considering the filter
    filtered_df_indecies_list = filtered_df.index
    project_df_indecies_list = project_df.index

    np.fill_diagonal(similarity_matrix, 0)
    match_matrix = similarity_matrix[filtered_df_indecies_list, :][:, project_df_indecies_list]

    best_matches_list = np.argsort(match_matrix, axis=None)

    if len(best_matches_list) < top_x:
        top_x = len(best_matches_list)

    # get row (project1) and column (project2) with highest similarity in filtered df
    top_indices = np.unravel_index(best_matches_list[-top_x:], match_matrix.shape)

    # get the corresponding similarity values
    top_values = match_matrix[top_indices]

    p1_df = filtered_df.iloc[top_indices[0]]
    p1_df["similarity"] = top_values
    p2_df = project_df.iloc[top_indices[1]]
    p2_df["similarity"] = top_values

    return p1_df, p2_df
"""

def calc_matches(filtered_df, project_df, similarity_matrix, top_x):
    # Ensure the matrix is in a suitable format for manipulation
    if not isinstance(similarity_matrix, csr_matrix):
        similarity_matrix = csr_matrix(similarity_matrix)

    # Get indices from dataframes
    filtered_df_indices = filtered_df.index.to_list()
    project_df_indices = project_df.index.to_list()

    # Efficiently zero out diagonal elements if necessary
    if np.array_equal(filtered_df_indices, project_df_indices):
        similarity_matrix = lil_matrix(similarity_matrix)
        similarity_matrix.setdiag(0)
        similarity_matrix = csr_matrix(similarity_matrix)

    # Select submatrix based on indices from both dataframes
    match_matrix = similarity_matrix[filtered_df_indices, :][:, project_df_indices]

    # Get the linear indices of the top 'top_x' values
    # (flattened index to handle the sparse matrix more effectively)
    linear_indices = np.argsort(match_matrix.data)[-top_x:]
    if len(linear_indices) < top_x:
        top_x = len(linear_indices)

    # Convert flat indices to 2D indices using the shape of the submatrix
    top_indices = np.unravel_index(linear_indices, match_matrix.shape)

    # Get the corresponding similarity values
    top_values = match_matrix.data[linear_indices]

    # Create resulting dataframes with top matches and their similarity scores
    p1_df = filtered_df.iloc[top_indices[0]].copy()
    p1_df['similarity'] = top_values
    p2_df = project_df.iloc[top_indices[1]].copy()
    p2_df['similarity'] = top_values

    return p1_df, p2_df