import pandas as pd import streamlit as st import datasets import plotly.express as px from sentence_transformers import SentenceTransformer from PIL import Image import os from pandas.api.types import ( is_categorical_dtype, is_datetime64_any_dtype, is_numeric_dtype, is_object_dtype, ) import subprocess from tempfile import NamedTemporaryFile from itertools import combinations import networkx as nx import plotly.graph_objects as go import colorcet as cc from matplotlib.colors import rgb2hex from sklearn.cluster import KMeans from sklearn.decomposition import PCA import hdbscan import umap import numpy as np from bokeh.plotting import figure from bokeh.models import ColumnDataSource from datetime import datetime #st.set_page_config(layout="wide") model_dir = "./models/sbert.net_models_sentence-transformers_clip-ViT-B-32-multilingual-v1" @st.cache_data(show_spinner=True) def download_models(): # Directory doesn't exist, download and extract the model subprocess.run(["mkdir", "models"]) subprocess.run(["wget", "--no-check-certificate", "https://public.ukp.informatik.tu-darmstadt.de/reimers/sentence-transformers/v0.2/clip-ViT-B-32-multilingual-v1.zip"], check=True) subprocess.run(["unzip", "-q", "clip-ViT-B-32-multilingual-v1.zip", "-d", model_dir], check=True) token = st.secrets["token"] @st.cache_data(show_spinner=True) def load_dataset(): dataset = datasets.load_dataset('rjadr/ditaduranuncamais', split='train', use_auth_token=token) dataset.add_faiss_index(column="txt_embs") dataset.add_faiss_index(column="img_embs") dataset = dataset.remove_columns(['Post Created Date', 'Post Created Time','Like and View Counts Disabled','Link','Download URL','Views']) return dataset @st.cache_data(show_spinner=False) def load_dataframe(_dataset): dataframe = _dataset.remove_columns(['txt_embs', 'img_embs']).to_pandas() # Extract hashtags ith regex and convert to set dataframe['Hashtags'] = dataframe.apply(lambda row: f"{row['Description']} {row['Image Text']}", axis=1) dataframe['Hashtags'] = dataframe['Hashtags'].str.lower().str.findall(r'#(\w+)').apply(set) # remove all hashtags that starts with 'throwback', 'thursday' or 'tbt' from the lists of hashtags per post # dataframe['Hashtags'] = dataframe['Hashtags'].apply(lambda x: [item for item in x if not item.startswith('ditaduranuncamais')]) # dataframe['Post Created'] = dataframe['Post Created'].dt.tz_convert('UTC') dataframe = dataframe[['Post Created', 'image', 'Description', 'Image Text', 'Account', 'User Name'] + [col for col in dataframe.columns if col not in ['Post Created', 'image', 'Description', 'Image Text', 'Account', 'User Name']]] return dataframe @st.cache_resource(show_spinner=True) def load_img_model(): # We use the original clip-ViT-B-32 for encoding images return SentenceTransformer('clip-ViT-B-32') @st.cache_resource(show_spinner=True) def load_txt_model(): # Our text embedding model is aligned to the img_model and maps 50+ # languages to the same vector space return SentenceTransformer('./models/sbert.net_models_sentence-transformers_clip-ViT-B-32-multilingual-v1') def filter_dataframe(df: pd.DataFrame) -> pd.DataFrame: """ Adds a UI on top of a dataframe to let viewers filter columns Args: df (pd.DataFrame): Original dataframe Returns: pd.DataFrame: Filtered dataframe """ modify = st.checkbox("Add filters") if not modify: return df df = df.copy() # Try to convert datetimes into a standard format (datetime, no timezone) for col in df.columns: if is_object_dtype(df[col]): try: df[col] = pd.to_datetime(df[col]) except Exception: pass if is_datetime64_any_dtype(df[col]): df[col] = df[col].dt.tz_localize(None) modification_container = st.container() with modification_container: to_filter_columns = st.multiselect("Filter dataframe on", df.columns) for column in to_filter_columns: left, right = st.columns((1, 20)) left.write("↳") # Treat columns with < 10 unique values as categorical if is_categorical_dtype(df[column]) or df[column].nunique() < 10: user_cat_input = right.multiselect( f"Values for {column}", df[column].unique(), default=list(df[column].unique()), ) df = df[df[column].isin(user_cat_input)] elif is_numeric_dtype(df[column]): _min = float(df[column].min()) _max = float(df[column].max()) step = (_max - _min) / 100 user_num_input = right.slider( f"Values for {column}", _min, _max, (_min, _max), step=step, ) df = df[df[column].between(*user_num_input)] elif is_datetime64_any_dtype(df[column]): user_date_input = right.date_input( f"Values for {column}", value=( df[column].min(), df[column].max(), ), ) if len(user_date_input) == 2: user_date_input = tuple(map(pd.to_datetime, user_date_input)) start_date, end_date = user_date_input df = df.loc[df[column].between(start_date, end_date)] else: user_text_input = right.text_input( f"Substring or regex in {column}", ) if user_text_input: df = df[df[column].str.contains(user_text_input)] return df @st.cache_data def get_image_embs(image): """ Get image embeddings Parameters: uploaded_file (PIL.Image): Uploaded image file Returns: img_emb (np.array): Image embeddings """ img_emb = image_model.encode(Image.open(image)) return img_emb @st.cache_data(show_spinner=False) def get_text_embs(text): """ Get text embeddings Parameters: text (str): Text to encode Returns: text_emb (np.array): Text embeddings """ txt_emb = text_model.encode(text) return txt_emb @st.cache_data def postprocess_results(scores, samples): """ Postprocess results to tuple of labels and scores Parameters: scores (np.array): Scores samples (datasets.Dataset): Samples Returns: labels (list): List of tuples of PIL images and labels/scores """ samples_df = pd.DataFrame.from_dict(samples) samples_df["score"] = scores samples_df["score"] = (1 - (samples_df["score"] - samples_df["score"].min()) / ( samples_df["score"].max() - samples_df["score"].min())) * 100 samples_df["score"] = samples_df["score"].astype(int) samples_df.reset_index(inplace=True, drop=True) samples_df = samples_df[['Post Created', 'image', 'Description', 'Image Text', 'Account', 'User Name'] + [col for col in samples_df.columns if col not in ['Post Created', 'image', 'Description', 'Image Text', 'Account', 'User Name']]] return samples_df.drop(columns=['txt_embs', 'img_embs']) @st.cache_data def text_to_text(text, k=5): """ Text to text Parameters: text (str): Input text k (int): Number of top results to return Returns: results (list): List of tuples of PIL images and labels/scores """ text_emb = get_text_embs(text) scores, samples = dataset.get_nearest_examples('txt_embs', text_emb, k=k) return postprocess_results(scores, samples) @st.cache_data def image_to_text(image, k=5): """ Image to text Parameters: image (str): Temp filepath to image k (int): Number of top results to return Returns: results (list): List of tuples of PIL images and labels/scores """ img_emb = get_image_embs(image.name) scores, samples = dataset.get_nearest_examples('txt_embs', img_emb, k=k) return postprocess_results(scores, samples) @st.cache_data def text_to_image(text, k=5): """ Text to image Parameters: text (str): Input text k (int): Number of top results to return Returns: results (list): List of tuples of PIL images and labels/scores """ text_emb = get_text_embs(text) scores, samples = dataset.get_nearest_examples('img_embs', text_emb, k=k) return postprocess_results(scores, samples) @st.cache_data def image_to_image(image, k=5): """ Image to image Parameters: image (str): Temp filepath to image k (int): Number of top results to return Returns: results (list): List of tuples of PIL images and labels/scores """ img_emb = get_image_embs(image.name) scores, samples = dataset.get_nearest_examples('img_embs', img_emb, k=k) return postprocess_results(scores, samples) def disparity_filter(g: nx.Graph, weight: str = 'weight', alpha: float = 0.05) -> nx.Graph: """ Computes the backbone of the input graph using the disparity filter algorithm. The algorithm is proposed in: M. A. Serrano, M. Boguna, and A. Vespignani, "Extracting the Multiscale Backbone of Complex Weighted Networks", PNAS, 106(16), pp 6483--6488 (2009). DOI: 10.1073/pnas.0808904106 Implementation taken from https://groups.google.com/g/networkx-discuss/c/bCuHZ3qQ2po/m/QvUUJqOYDbIJ Parameters ---------- g : NetworkX graph The input graph. weight : str, optional (default='weight') The name of the edge attribute to use as weight. alpha : float, optional (default=0.05) The statistical significance level for the disparity filter (p-value). Returns ------- backbone_graph : NetworkX graph The backbone graph. """ # Create an empty graph for the backbone backbone_graph = nx.Graph() # Iterate over all nodes in the input graph for node in g: # Get the degree of the node (number of edges connected to the node) k_n = len(g[node]) # Only proceed if the node has more than one connection if k_n > 1: # Calculate the sum of weights of edges connected to the node sum_w = sum(g[node][neighbor][weight] for neighbor in g[node]) # Iterate over all neighbors of the node for neighbor in g[node]: # Get the weight of the edge between the node and its neighbor edge_weight = g[node][neighbor][weight] # Calculate the proportion of the total weight that this edge represents pij = float(edge_weight) / sum_w # Perform the disparity filter test. If it passes, the edge is considered significant and is added to the backbone if (1 - pij) ** (k_n - 1) < alpha: backbone_graph.add_edge(node, neighbor, weight=edge_weight) # Return the backbone graph return backbone_graph st.cache_data(show_spinner=True) def assign_community_colors(G: nx.Graph, attr: str = 'community') -> dict: """ Assigns a unique color to each community in the input graph. Parameters ---------- G : nx.Graph The input graph. attr : str, optional The node attribute of the community names or indexes (default is 'community'). Returns ------- dict A dictionary mapping each community to a unique color. """ glasbey_colors = cc.glasbey_hv communities_ = set(nx.get_node_attributes(G, attr).values()) return {community: rgb2hex(glasbey_colors[i % len(glasbey_colors)]) for i, community in enumerate(communities_)} st.cache_data(show_spinner=True) def generate_hover_text(G: nx.Graph, attr: str = 'community') -> list: """ Generates hover text for each node in the input graph. Parameters ---------- G : nx.Graph The input graph. attr : str, optional The node attribute of the community names or indexes (default is 'community'). Returns ------- list A list of strings containing the hover text for each node. """ return [f"Node: {str(node)}
Community: {G.nodes[node][attr] + 1}
# of connections: {len(adjacencies)}" for node, adjacencies in G.adjacency()] st.cache_data(show_spinner=True) def calculate_node_sizes(G: nx.Graph) -> list: """ Calculates the size of each node in the input graph based on its degree. Parameters ---------- G : nx.Graph The input graph. Returns ------- list A list of node sizes. """ degrees = dict(G.degree()) max_degree = max(deg for node, deg in degrees.items()) return [10 + 20 * (degrees[node] / max_degree) for node in G.nodes()] @st.cache_data(show_spinner=True) def plot_graph(_G: nx.Graph, layout: str = "fdp", community_names_lookup: dict = None): """ Plots a network graph with communities. Parameters ---------- G : nx.Graph The input graph. layout : str, optional The layout algorithm to use (default is "fdp"). """ pos = nx.spring_layout(G_backbone, dim=3, seed=779) community_colors = assign_community_colors(_G) node_colors = [community_colors[_G.nodes[n]['community']] for n in _G.nodes] edge_trace = go.Scatter(x=[item for sublist in [[pos[edge[0]][0], pos[edge[1]][0], None] for edge in _G.edges()] for item in sublist], y=[item for sublist in [[pos[edge[0]][1], pos[edge[1]][1], None] for edge in _G.edges()] for item in sublist], line=dict(width=0.5, color='#888'), hoverinfo='none', mode='lines') node_trace = go.Scatter(x=[pos[n][0] for n in _G.nodes()], y=[pos[n][1] for n in _G.nodes()], mode='markers', hoverinfo='text', marker=dict(color=node_colors, size=10, line_width=2)) node_trace.text = generate_hover_text(_G) node_trace.marker.size = calculate_node_sizes(_G) fig = go.Figure(data=[edge_trace, node_trace], layout=go.Layout(title='Network graph with communities', titlefont=dict(size=16), showlegend=False, hovermode='closest', margin=dict(b=20,l=5,r=5,t=40), xaxis=dict(showgrid=False, zeroline=False, showticklabels=False), yaxis=dict(showgrid=False, zeroline=False, showticklabels=False), height=800)) # Extract node positions Xn=[pos[k][0] for k in G_backbone.nodes()] # x-coordinates of nodes Yn=[pos[k][1] for k in G_backbone.nodes()] # y-coordinates Zn=[pos[k][2] for k in G_backbone.nodes()] # z-coordinates # Extract edge positions Xe=[] Ye=[] Ze=[] for e in G_backbone.edges(): Xe+=[pos[e[0]][0],pos[e[1]][0], None] # x-coordinates of edge ends Ye+=[pos[e[0]][1],pos[e[1]][1], None] Ze+=[pos[e[0]][2],pos[e[1]][2], None] # Define traces for plotly trace1=go.Scatter3d(x=Xe, y=Ye, z=Ze, mode='lines', line=dict(color='rgb(125,125,125)', width=1), hoverinfo='none' ) # Map community numbers to names community_names = {i: community_names_lookup[f"Community {i+1}"] for i in range(len(communities))} # Create hover text hover_text = [f"{node} ({community_names[G_backbone.nodes[node]['community']]})" for node in G_backbone.nodes()] trace2=go.Scatter3d(x=Xn, y=Yn, z=Zn, mode='markers', name='actors', marker=dict(symbol='circle', size=7, color=node_colors, # pass hex colors line=dict(color='rgb(50,50,50)', width=0.2) ), text=hover_text, # Use community names as hover text hoverinfo='text' ) axis=dict(showbackground=False, showline=False, zeroline=False, showgrid=False, showticklabels=False, title='' ) layout = go.Layout( title="3D Network Graph", width=1000, height=1000, showlegend=False, scene=dict( xaxis=dict(axis), yaxis=dict(axis), zaxis=dict(axis), ), margin=dict( t=100 ), hovermode='closest', ) data=[trace1, trace2] fig=go.Figure(data=data, layout=layout) return fig @st.cache_data(show_spinner=True) def cluster_embeddings(embeddings, clustering_algo='KMeans', dim_reduction='PCA', n_clusters=5, min_cluster_size=5, n_components=2, n_neighbors=15, min_dist=0.0, random_state=42, min_samples=5): """ A function to cluster embeddings. Args: embeddings (pd.Series): A series of numpy vectors. clustering_algo (str): The clustering algorithm to use. Either 'KMeans' or 'HDBSCAN'. dim_reduction (str): The dimensionality reduction method to use. Either 'PCA' or 'UMAP'. n_clusters (int): The number of clusters for KMeans. min_cluster_size (int): The minimum cluster size for HDBSCAN. n_components (int): The number of components for the dimensionality reduction method. n_neighbors (int): The number of neighbors for UMAP. min_dist (float): The minimum distance for UMAP. random_state (int): The seed used by the random number generator. min_samples (int): The minimum number of samples for HDBSCAN. Returns: pd.Series: A series of cluster labels. """ # Dimensionality reduction if dim_reduction == 'PCA': reducer = PCA(n_components=n_components, random_state=random_state) elif dim_reduction == 'UMAP': reducer = umap.UMAP(n_neighbors=n_neighbors, min_dist=min_dist, n_components=n_components, random_state=random_state) else: raise ValueError('Invalid dimensionality reduction method') reduced_embeddings = reducer.fit_transform(np.stack(embeddings)) # Clustering if clustering_algo == 'KMeans': clusterer = KMeans(n_clusters=n_clusters, random_state=random_state) elif clustering_algo == 'HDBSCAN': clusterer = hdbscan.HDBSCAN(min_cluster_size=min_cluster_size, min_samples=min_samples) else: raise ValueError('Invalid clustering algorithm') labels = clusterer.fit_predict(reduced_embeddings) return labels, reduced_embeddings st.title("#ditaduranuncamais Data Explorer") def check_password(): """Returns `True` if the user had the correct password.""" def password_entered(): """Checks whether a password entered by the user is correct.""" if st.session_state["password"] == st.secrets["password"]: st.session_state["password_correct"] = True del st.session_state["password"] # don't store password else: st.session_state["password_correct"] = False if "password_correct" not in st.session_state: # First run, show input for password. st.text_input( "Password", type="password", on_change=password_entered, key="password" ) return False elif not st.session_state["password_correct"]: # Password not correct, show input + error. st.text_input( "Password", type="password", on_change=password_entered, key="password" ) st.error("😕 Password incorrect") return False else: # Password correct. return True if not check_password(): st.stop() # Check if the directory exists if not os.path.exists(model_dir): download_models() dataset = load_dataset() df = load_dataframe(dataset) image_model = load_img_model() text_model = load_txt_model() menu_options = ["Data exploration", "Semantic search", "Hashtags", "Clustering", "Stats"] st.sidebar.markdown('# Menu') selected_menu_option = st.sidebar.radio("Select a page", menu_options) if selected_menu_option == "Data exploration": st.dataframe( data=filter_dataframe(df), # use_container_width=True, column_config={ "image": st.column_config.ImageColumn( "Image", help="Instagram image" ), "URL": st.column_config.LinkColumn( "Link", help="Instagram link", width="small" ) }, hide_index=True, ) elif selected_menu_option == "Semantic search": tabs = ["Text to Text", "Text to Image", "Image to Image", "Image to Text"] selected_tab = st.sidebar.radio("Select a search type", tabs) if selected_tab == "Text to Text": st.markdown('## Text to text search') text_to_text_input = st.text_input("Enter text") text_to_text_k_top = st.slider("Number of results", 1, 500, 20) if st.button("Search"): if not text_to_text_input: st.warning("Please enter text") else: st.dataframe( data=text_to_text(text_to_text_input, text_to_text_k_top), column_config={ "image": st.column_config.ImageColumn( "Image", help="Instagram image" ), "URL": st.column_config.LinkColumn( "Link", help="Instagram link", width="small" ) }, hide_index=True, ) elif selected_tab == "Text to Image": st.markdown('## Text to image search') text_to_image_input = st.text_input("Enter text") text_to_image_k_top = st.slider("Number of results", 1, 500, 20) if st.button("Search"): if not text_to_image_input: st.warning("Please enter some text") else: st.dataframe( data=text_to_image(text_to_image_input, text_to_image_k_top), column_config={ "image": st.column_config.ImageColumn( "Image", help="Instagram image" ), "URL": st.column_config.LinkColumn( "Link", help="Instagram link", width="small" ) }, hide_index=True, ) elif selected_tab == "Image to Image": st.markdown('## Image to image search') image_to_image_k_top = st.slider("Number of results", 1, 500, 20) image_to_image_input = st.file_uploader("Upload an image", type=["jpg", "jpeg", "png"]) temp_file = NamedTemporaryFile(delete=False) if st.button("Search"): if not image_to_image_input: st.warning("Please upload an image") else: temp_file.write(image_to_image_input.getvalue()) st.dataframe( data=image_to_image(temp_file, image_to_image_k_top), column_config={ "image": st.column_config.ImageColumn( "Image", help="Instagram image" ), "URL": st.column_config.LinkColumn( "Link", help="Instagram link", width="small" ) }, hide_index=True, ) elif selected_tab == "Image to Text": st.markdown('## Image to text search') image_to_text_k_top = st.slider("Number of results", 1, 500, 20) image_to_text_input = st.file_uploader("Upload an image", type=["jpg", "jpeg", "png"]) temp_file = NamedTemporaryFile(delete=False) if st.button("Search"): if not image_to_text_input: st.warning("Please upload an image") else: temp_file.write(image_to_text_input.getvalue()) st.dataframe( data=image_to_text(temp_file, image_to_text_k_top), column_config={ "image": st.column_config.ImageColumn( "Image", help="Instagram image" ), "URL": st.column_config.LinkColumn( "Link", help="Instagram link", width="small" ) }, hide_index=True, ) elif selected_menu_option == "Hashtags": if 'dfx' not in st.session_state: st.session_state.dfx = df.copy() # Make a copy of dfx # Get a list of all unique hashtags in the DataFrame all_hashtags = list(set([item for sublist in st.session_state.dfx['Hashtags'].tolist() for item in sublist])) st.sidebar.markdown('# Hashtag co-occurrence analysis options') # Let users select hashtags to remove hashtags_to_remove = st.sidebar.multiselect("Hashtags to remove", all_hashtags) col1, col2 = st.sidebar.columns(2) # Add a button to trigger the removal operation if col1.button("Remove hashtags"): # If dfx does not exist in session state, create it st.session_state.dfx['Hashtags'] = st.session_state.dfx['Hashtags'].apply(lambda x: [item for item in x if item not in hashtags_to_remove]) # Add a reset button if col2.button("Reset"): st.session_state.dfx = df.copy() # Reset dfx to the original DataFrame # Count the number of unique hashtags hashtags = [item for sublist in st.session_state.dfx['Hashtags'].tolist() for item in sublist] # Count the number of posts per hashtag hashtag_freq = st.session_state.dfx.explode('Hashtags').groupby('Hashtags').size().reset_index(name='counts') # Sort the hashtags by frequency hashtag_freq = hashtag_freq.sort_values(by='counts', ascending=False) # Make the scatter plot hashtags_fig = px.scatter(hashtag_freq, x='Hashtags', y='counts', log_y=True, # Set log_y to True to make the plot more readable on a log scale labels={'Hashtags': 'Hashtags', 'counts': 'Frequency'}, title='Frequency of hashtags in #throwbackthursday posts on Instagram', height=600) # Set the height to 600 pixels st.markdown("### Hashtag Frequency Distribution") st.markdown('Here we apply hashtag co-occurence analysis for mnemonic community detection. This detects communities through creating a network of hashtag pairs (which hashtags are used together in which posts) and then applying community detection algorithms on this network.') st.plotly_chart(hashtags_fig) weight_option = st.sidebar.radio( 'Select weight definition', ('Number of users that use the hashtag pairs', 'Total number of occurrences') ) hashtag_user_pairs = [(tuple(sorted(combination)), userid) for hashtags, userid in zip(st.session_state.dfx['Hashtags'], st.session_state.dfx['User Name']) for combination in combinations(hashtags, r=2)] # Create a DataFrame with columns 'hashtag_pair' and 'userid' hashtag_user_df = pd.DataFrame(hashtag_user_pairs, columns=['hashtag_pair', 'User Name']) if weight_option == 'Number of users that use the hashtag pairs': # Group by 'hashtag_pair' and count the number of unique 'userid's hashtag_user_df = hashtag_user_df.groupby('hashtag_pair').agg({'User Name': 'nunique'}).reset_index() elif weight_option == 'Total number of occurrences': # Group by 'hashtag_pair' and count the total number of occurrences hashtag_user_df = hashtag_user_df.groupby('hashtag_pair').size().reset_index(name='User Name') # Make edge_list from hashtag_user_df with columns 'hashtag1', 'hashtag2', and 'weight' edge_list = hashtag_user_df.rename(columns={'hashtag_pair': 'hashtag1', 'User Name': 'weight'}) edge_list[['hashtag1', 'hashtag2']] = pd.DataFrame(edge_list['hashtag1'].tolist(), index=edge_list.index) edge_list = edge_list[['hashtag1', 'hashtag2', 'weight']] st.markdown("### Edge List of Hashtag Pairs") # Create the graph using the unique users as adge attributes G = nx.from_pandas_edgelist(edge_list, 'hashtag1', 'hashtag2', 'weight') G_backbone = disparity_filter(G, weight='weight', alpha=0.05) st.markdown(f'Number of nodes {len(G_backbone.nodes)}') st.markdown(f'Number of edges {len(G_backbone.edges)}') st.dataframe(edge_list.sort_values(by='weight', ascending=False).head(10).style.set_caption("Edge list of hashtag pairs with the highest weight")) # Create louvain communities communities = nx.community.louvain_communities(G_backbone, weight='weight', seed=1234) communities = list(communities) # Sort communities by size communities.sort(key=len, reverse=True) for i, community in enumerate(communities): for node in community: G_backbone.nodes[node]['community'] = i # Sort community hashtags based on their weighted degree in the network sorted_community_hashtags = [ [ hashtag for hashtag, degree in sorted( ((h, G.degree(h, weight='weight')) for h in community), key=lambda x: x[1], reverse=True ) ] for community in communities ] # Convert the sorted_community_hashtags list into a DataFrame and transpose it sorted_community_hashtags = pd.DataFrame(sorted_community_hashtags).T # Rename the columns of sorted_community_hashtags DataFrame sorted_community_hashtags.columns = [f'Community {i+1}' for i in range(len(sorted_community_hashtags.columns))] st.markdown("### Hashtag Communities") st.markdown(f'There are {len(communities)} communities in the graph.') st.dataframe(sorted_community_hashtags) # add a st.data_editor with Community 1, etc as index and a column "community names" that sets Community 1 etc as default value st.markdown("### Community Names") st.markdown("Edit the names of the communities in the table below so they show up in the visualisations.") df_community_names = pd.DataFrame(sorted_community_hashtags.columns, columns=['community_names'], index=sorted_community_hashtags.columns) df_community_names = st.data_editor(df_community_names) #create dict with community names community_names_lookup = df_community_names['community_names'].to_dict() # implement time series analysis of size of communities over time using resample_dict st.markdown("### Community Size Over Time") st.markdown("Select communites to see their size over time.") # selected_communities = st.multiselect('Select Communities', [f'Community {i+1}' for i in range(len(communities))], default=[f'Community {i+1}' for i in range(len(communities))]) selected_communities = st.multiselect('Select Communities', community_names_lookup.values(), default=community_names_lookup.values()) # Dropdown to select time resampling resample_dict = { 'Day': 'D', 'Three Days': '3D', 'Week': 'W', 'Two Weeks': '2W', 'Month': 'M', 'Quarter': 'Q', 'Year': 'Y' } # Dropdown to select time resampling resample_time = st.selectbox('Select Time Resampling', list(resample_dict.keys()), index=4) df_communities = st.session_state.dfx.copy() def community_dict(communities): community_dict = {} for i, community in enumerate(communities): for node in community: community_dict[node] = community_names_lookup[f'Community {i+1}'] return community_dict community_dict = community_dict(communities) df_communities['Communities'] = df_communities['Hashtags'].apply(lambda x: [community_dict[tag] for tag in x if tag in community_dict.keys()]) df_communities = df_communities[['Post Created', 'Communities']].explode('Communities') df_communities = df_communities.dropna(subset=['Communities']) # Slider for date range selection min_date = df_communities['Post Created'].min().date() max_date = df_communities['Post Created'].max().date() date_range = st.slider('Select Date Range', min_value=min_date, max_value=max_date, value=(min_date, max_date)) # Filter df_communities by the selected date range df_communities = df_communities[(df_communities['Post Created'].dt.date >= date_range[0]) & (df_communities['Post Created'].dt.date <= date_range[1])] # Count the number of posts per community per resample_time df_communities['Post Created'] = df_communities['Post Created'].dt.to_period(resample_dict[resample_time]) df_community_sizes = df_communities.groupby(['Post Created', 'Communities']).size().unstack(fill_value=0) df_community_sizes.index = df_community_sizes.index.to_timestamp() # Filter the DataFrame to include only the selected communities df_community_sizes = df_community_sizes[selected_communities] st.plotly_chart(px.line(df_community_sizes, title='Community Size Over Time', labels={'value': 'Number of posts', 'index': 'Date', 'variable': 'Community'})) st.markdown("### Hashtag Network Graph") st.plotly_chart(plot_graph(G_backbone, layout="fdp", community_names_lookup=community_names_lookup)) # fdp is relatively slow, use 'sfdp' or 'neato' for faster but denser layouts elif selected_menu_option == "Clustering": st.markdown("## Clustering") st.markdown("Select the type of embeddings to cluster and the clustering algorithm and dimensionality reduction method to use in the sidebar. Then click run clustering. Clustering may take some time.") st.sidebar.markdown("# Clustering Options") type_embeddings = st.sidebar.selectbox("Type of embeddings to cluster", ["Text", "Image"]) clustering_algo = st.sidebar.selectbox("Clustering algorithm", ["HDBSCAN", "KMeans"]) dim_reduction = st.sidebar.selectbox("Dimensionality reduction method", ["UMAP", "PCA"]) if clustering_algo == "KMeans": st.sidebar.markdown("### KMeans Options") n_clusters = st.sidebar.slider("Number of clusters", 2, 20, 5) min_cluster_size = None min_samples = None elif clustering_algo == "HDBSCAN": st.sidebar.markdown("### HDBSCAN Options") min_cluster_size = st.sidebar.slider("[Minimum cluster size](https://hdbscan.readthedocs.io/en/latest/parameter_selection.html#selecting-min-cluster-size)", 2, 200, 5) min_samples = st.sidebar.slider("[Minimum samples](https://hdbscan.readthedocs.io/en/latest/parameter_selection.html#selecting-min-samples)", 2, 50, 5) n_clusters = None if dim_reduction == "UMAP": st.sidebar.markdown("### UMAP Options") n_components = st.sidebar.slider("[Number of components](https://umap-learn.readthedocs.io/en/latest/parameters.html#n-components)", 2, 80, 50) n_neighbors = st.sidebar.slider("[Number of neighbors](https://umap-learn.readthedocs.io/en/latest/parameters.html#n-neighbors)", 2, 20, 15) min_dist = st.sidebar.slider("[Minimum distance](https://umap-learn.readthedocs.io/en/latest/parameters.html#min-dist)", 0.0, 1.0, 0.0) else: st.sidebar.markdown("### PCA Options") n_components = st.sidebar.slider("[Number of components](https://scikit-learn.org/stable/modules/generated/sklearn.decomposition.PCA.html)", 2, 80, 2) n_neighbors = None min_dist = None st.markdown("### Clustering Results") if type_embeddings == "Text": embeddings = dataset['txt_embs'] elif type_embeddings == "Image": embeddings = dataset['img_embs'] # Cluster embeddings labels, reduced_embeddings = cluster_embeddings(embeddings, clustering_algo=clustering_algo, dim_reduction=dim_reduction, n_clusters=n_clusters, min_cluster_size=min_cluster_size, n_components=n_components, n_neighbors=n_neighbors, min_dist=min_dist) st.markdown(f"Clustering {type_embeddings} embeddings using {clustering_algo} with {dim_reduction} dimensionality reduction method resulting in **{len(set(labels))}** clusters.") df_clustered = df.copy() df_clustered['cluster'] = labels df_clustered = df_clustered.set_index('cluster').reset_index() st.dataframe( data=filter_dataframe(df_clustered), # use_container_width=True, column_config={ "image": st.column_config.ImageColumn( "Image", help="Instagram image" ), "URL": st.column_config.LinkColumn( "Link", help="Instagram link", width="small" ) }, hide_index=True, ) st.download_button( "Download dataset with labels", df_clustered.to_csv(index=False).encode('utf-8'), f'ditaduranuncamais_{datetime.now().strftime("%Y%m%d-%H%M%S")}.csv', "text/csv", key='download-csv' ) st.markdown("### Cluster Plot") # Plot the scatter plot in plotly with the cluster labels as colors reduce further to 2 dimensions if n_components > 2 if n_components > 2: reducer = umap.UMAP(n_components=2, random_state=42) reduced_embeddings = reducer.fit_transform(reduced_embeddings) # set the labels to be the cluster labels dynamically # visualise with bokeh showing df_clustered['Description'] and df_clustered['image'] on hover descriptions = df_clustered['Description'].tolist() images = df_clustered['image'].tolist() glasbey_colors = cc.glasbey_hv color_dict = {n: rgb2hex(glasbey_colors[i % len(glasbey_colors)]) for i, n in enumerate(set(labels))} colors = [color_dict[label] for label in labels] source = ColumnDataSource(data=dict( x=reduced_embeddings[:, 0], y=reduced_embeddings[:, 1], desc=descriptions, imgs=images, colors=colors )) TOOLTIPS = """
@imgs
@desc
""" p = figure(width=800, height=800, tooltips=TOOLTIPS, title="Mouse over the dots") p.circle('x', 'y', size=10, source=source, color='colors', line_color=None) st.bokeh_chart(p) # inster time series graph for clusters sorted by size (except cluster -1, show top5 by default, but include selectbox. reuse resample_dict for binning) st.markdown("### Cluster Size") cluster_sizes = df_clustered.groupby('cluster').size().reset_index(name='counts') cluster_sizes = cluster_sizes.sort_values(by='counts', ascending=False) cluster_sizes = cluster_sizes[cluster_sizes['cluster'] != -1] cluster_sizes = cluster_sizes.set_index('cluster').reset_index() cluster_sizes = cluster_sizes.rename(columns={'cluster': 'Cluster', 'counts': 'Size'}) st.dataframe(cluster_sizes) st.markdown("### Cluster Time Series") # Dropdown to select variables variable = st.selectbox('Select Variable', ['Likes', 'Comments', 'Followers at Posting', 'Total Interactions']) # Dropdown to select time resampling resample_dict = { 'Day': 'D', 'Three Days': '3D', 'Week': 'W', 'Two Weeks': '2W', 'Month': 'M', 'Quarter': 'Q', 'Year': 'Y' } # Dropdown to select time resampling resample_time = st.selectbox('Select Time Resampling', list(resample_dict.keys())) # Slider for date range selection min_date = df_clustered['Post Created'].min().date() max_date = df_clustered['Post Created'].max().date() date_range = st.slider('Select Date Range', min_value=min_date, max_value=max_date, value=(min_date, max_date)) # Filter dataframe based on selected date range df_resampled = df_clustered[(df_clustered['Post Created'].dt.date >= date_range[0]) & (df_clustered['Post Created'].dt.date <= date_range[1])] df_resampled = df_resampled.set_index('Post Created') # Get unique clusters and their sizes cluster_sizes = df_resampled[df_resampled['cluster'] != -1]['cluster'].value_counts() clusters = cluster_sizes.index # Select the largest 5 clusters by default default_clusters = cluster_sizes.sort_values(ascending=False).head(5).index.tolist() # Multiselect widget to choose clusters selected_clusters = st.multiselect('Select Clusters', options=clusters.tolist(), default=default_clusters) # Create a new DataFrame for the plot df_plot = pd.DataFrame() # Loop through selected clusters for cluster in selected_clusters: # Create a separate DataFrame for each cluster, resample and add to the plot DataFrame df_cluster = df_resampled[df_resampled['cluster'] == cluster][variable].resample(resample_dict[resample_time]).sum() df_plot = pd.concat([df_plot, df_cluster], axis=1) # Add legend (use cluster numbers as legend) df_plot.columns = selected_clusters # Create the line chart st.line_chart(df_plot) elif selected_menu_option == "Stats": st.markdown("### Time Series Analysis") # Dropdown to select variables variable = st.selectbox('Select Variable', ['Followers at Posting', 'Total Interactions', 'Likes', 'Comments']) # Dropdown to select time resampling resample_dict = { 'Day': 'D', 'Three Days': '3D', 'Week': 'W', 'Two Weeks': '2W', 'Month': 'M', 'Quarter': 'Q', 'Year': 'Y' } # Dropdown to select time resampling resample_time = st.selectbox('Select Time Resampling', list(resample_dict.keys())) df_filtered = df.set_index('Post Created') # Slider for date range selection min_date = df_filtered.index.min().date() max_date = df_filtered.index.max().date() date_range = st.slider('Select Date Range', min_value=min_date, max_value=max_date, value=(min_date, max_date)) # Filter dataframe based on selected date range df_filtered = df_filtered[(df_filtered.index.date >= date_range[0]) & (df_filtered.index.date <= date_range[1])] # Create a separate DataFrame for resampling and plotting df_resampled = df_filtered[variable].resample(resample_dict[resample_time]).sum() st.line_chart(df_resampled) st.markdown("### Correlation Analysis") # Dropdown to select variables for scatter plot options = ['Followers at Posting', 'Total Interactions', 'Likes', 'Comments'] scatter_variable_1 = st.selectbox('Select Variable 1 for Scatter Plot', options) # options.remove(scatter_variable_1) # remove the chosen option from the list scatter_variable_2 = st.selectbox('Select Variable 2 for Scatter Plot', options) # Plot scatter chart st.write(f"Scatter Plot of {scatter_variable_1} vs {scatter_variable_2}") # Plot scatter chart scatter_fig = px.scatter(df_filtered, x=scatter_variable_1, y=scatter_variable_2) #, trendline='ols', trendline_color_override='red') st.plotly_chart(scatter_fig) # calculate correlation for scatter_variable_1 with scatter_variable_2 corr = df_filtered[scatter_variable_1].corr(df_filtered[scatter_variable_2]) if corr > 0.7: st.write(f"The correlation coefficient is {corr}, indicating a strong positive relationship between {scatter_variable_1} and {scatter_variable_2}.") elif corr > 0.3: st.write(f"The correlation coefficient is {corr}, indicating a moderate positive relationship between {scatter_variable_1} and {scatter_variable_2}.") elif corr > -0.3: st.write(f"The correlation coefficient is {corr}, indicating a weak or no relationship between {scatter_variable_1} and {scatter_variable_2}.") elif corr > -0.7: st.write(f"The correlation coefficient is {corr}, indicating a moderate negative relationship between {scatter_variable_1} and {scatter_variable_2}.") else: st.write(f"The correlation coefficient is {corr}, indicating a strong negative relationship between {scatter_variable_1} and {scatter_variable_2}.")