import streamlit as st from pyvis.network import Network import plotly.express as px from sklearn.metrics.pairwise import cosine_similarity from sentence_transformers import SentenceTransformer from bertopic import BERTopic from sklearn.feature_extraction.text import CountVectorizer import pandas as pd import numpy as np import networkx as nx def reset_default_topic_sliders(min_topic_size, n_gram_range): st.session_state['min_topic_size'] = min_topic_size st.session_state['n_gram_range'] = n_gram_range def reset_default_threshold_slider(threshold): st.session_state['threshold'] = threshold @st.cache(allow_output_mutation=True) def load_sbert_model(): return SentenceTransformer('allenai-specter') @st.cache() def load_data(uploaded_file): data = pd.read_csv(uploaded_file) data = data[['Title', 'Abstract']] data = data.dropna() data = data.reset_index(drop=True) return data @st.cache(allow_output_mutation=True) def topic_modeling(data, min_topic_size, n_gram_range): """Topic modeling using BERTopic """ topic_model = BERTopic( embedding_model=load_sbert_model(), vectorizer_model=CountVectorizer( stop_words='english', ngram_range=n_gram_range), min_topic_size=min_topic_size ) # For 'allenai-specter' data['Title + Abstract'] = data['Title'] + '[SEP]' + data['Abstract'] # Train the topic model data["Topic"], data["Probs"] = topic_model.fit_transform( data['Title + Abstract']) # Merge topic results topic_df = topic_model.get_topic_info()[['Topic', 'Name']] data = data.merge(topic_df, on='Topic', how='left') # Topics topics = topic_df.set_index('Topic').to_dict(orient='index') return data, topic_model, topics @st.cache(allow_output_mutation=True) def embeddings(data): data['embedding'] = load_sbert_model().encode( data['Title + Abstract']).tolist() return data @st.cache() def cosine_sim(data): cosine_sim_matrix = cosine_similarity(data['embedding'].values.tolist()) # Take only upper triangular matrix cosine_sim_matrix = np.triu(cosine_sim_matrix, k=1) return cosine_sim_matrix @st.cache() def calc_max_connections(num_papers, ratio): n = ratio*num_papers return n*(n-1)/2 @st.cache() def calc_optimal_threshold(cosine_sim_matrix, max_connections): """Calculates the optimal threshold for the cosine similarity matrix. Allows a max of max_connections """ thresh_sweep = np.arange(0.05, 1.05, 0.05) for idx, threshold in enumerate(thresh_sweep): neighbors = np.argwhere(cosine_sim_matrix >= threshold).tolist() if len(neighbors) < max_connections: break return round(thresh_sweep[idx-1], 2).item(), round(thresh_sweep[idx], 2).item() @st.cache() def calc_neighbors(cosine_sim_matrix, threshold): neighbors = np.argwhere(cosine_sim_matrix >= threshold).tolist() return neighbors, len(neighbors) def nx_hash_func(nx_net): """Hash function for NetworkX graphs. """ return (list(nx_net.nodes()), list(nx_net.edges())) def pyvis_hash_func(pyvis_net): """Hash function for pyvis graphs. """ return (pyvis_net.nodes, pyvis_net.edges) @st.cache(hash_funcs={nx.Graph: nx_hash_func, Network: pyvis_hash_func}) def network_plot(data, topics, neighbors): """Creates a network plot of connected papers. Colored by Topic Model topics. """ nx_net = nx.Graph() pyvis_net = Network(height='750px', width='100%', bgcolor='#222222') # Add Nodes nodes = [ ( row.Index, { 'group': row.Topic, 'label': row.Index, 'title': row.Title, 'size': 20, 'font': {'size': 20, 'color': 'white'} } ) for row in data.itertuples() ] nx_net.add_nodes_from(nodes) assert(nx_net.number_of_nodes() == len(data)) # Add Legend Nodes step = 150 x = -2000 y = -500 legend_nodes = [ ( len(data)+idx, { 'group': key, 'label': ', '.join(value['Name'].split('_')[1:]), 'size': 30, 'physics': False, 'x': x, 'y': f'{y + idx*step}px', # , 'fixed': True, 'shape': 'box', 'widthConstraint': 1000, 'font': {'size': 40, 'color': 'black'} } ) for idx, (key, value) in enumerate(topics.items()) ] nx_net.add_nodes_from(legend_nodes) # Add Edges nx_net.add_edges_from(neighbors) assert(nx_net.number_of_edges() == len(neighbors)) # Plot the Pyvis graph pyvis_net.from_nx(nx_net) return nx_net, pyvis_net @st.cache() def network_centrality(data, centrality, centrality_option): """Calculates the centrality of the network """ # Sort Top 10 Central nodes central_nodes = sorted( centrality.items(), key=lambda item: item[1], reverse=True) central_nodes = pd.DataFrame(central_nodes, columns=[ 'node', centrality_option]).set_index('node') joined_data = data.join(central_nodes) top_central_nodes = joined_data.sort_values( centrality_option, ascending=False).head(10) # Plot the Top 10 Central nodes fig = px.bar(top_central_nodes, x=centrality_option, y='Title') fig.update_layout(yaxis={'categoryorder': 'total ascending'}, font={'size': 15}, height=800, width=800) return fig