Spaces:

rjadr
/

ditaduranuncamais_explorer

Sleeping

App Files Files Community

ditaduranuncamais_explorer / app.py

rjadr

Update app.py

42da8d1 verified 12 days ago

raw history blame contribute delete

No virus

45.1 kB

	import pandas as pd
	import streamlit as st
	import datasets
	import plotly.express as px
	from sentence_transformers import SentenceTransformer
	from PIL import Image
	import os
	from pandas.api.types import (
	is_categorical_dtype,
	is_datetime64_any_dtype,
	is_numeric_dtype,
	is_object_dtype,
	)
	import subprocess
	from tempfile import NamedTemporaryFile
	from itertools import combinations
	import networkx as nx
	import plotly.graph_objects as go
	import colorcet as cc
	from matplotlib.colors import rgb2hex
	from sklearn.cluster import KMeans
	from sklearn.decomposition import PCA
	import hdbscan
	import umap
	import numpy as np
	from bokeh.plotting import figure
	from bokeh.models import ColumnDataSource
	from datetime import datetime

	#st.set_page_config(layout="wide")

	model_dir = "./models/sbert.net_models_sentence-transformers_clip-ViT-B-32-multilingual-v1"

	@st.cache_data(show_spinner=True)
	def download_models():
	# Directory doesn't exist, download and extract the model
	subprocess.run(["mkdir", "models"])
	subprocess.run(["wget", "--no-check-certificate", "https://public.ukp.informatik.tu-darmstadt.de/reimers/sentence-transformers/v0.2/clip-ViT-B-32-multilingual-v1.zip"], check=True)
	subprocess.run(["unzip", "-q", "clip-ViT-B-32-multilingual-v1.zip", "-d", model_dir], check=True)

	token_ = st.secrets["token"]

	@st.cache_data(show_spinner=True)
	def load_dataset():
	dataset = datasets.load_dataset('rjadr/ditaduranuncamais', split='train', use_auth_token=token_)
	dataset.add_faiss_index(column="txt_embs")
	dataset.add_faiss_index(column="img_embs")
	dataset = dataset.remove_columns(['Post Created Date', 'Post Created Time','Like and View Counts Disabled','Link','Download URL','Views'])
	return dataset

	@st.cache_data(show_spinner=False)
	def load_dataframe(_dataset):
	dataframe = _dataset.remove_columns(['txt_embs', 'img_embs']).to_pandas()
	# Extract hashtags ith regex and convert to set
	dataframe['Hashtags'] = dataframe.apply(lambda row: f"{row['Description']} {row['Image Text']}", axis=1)
	dataframe['Hashtags'] = dataframe['Hashtags'].str.lower().str.findall(r'#(\w+)').apply(set)
	# remove all hashtags that starts with 'throwback', 'thursday' or 'tbt' from the lists of hashtags per post
	# dataframe['Hashtags'] = dataframe['Hashtags'].apply(lambda x: [item for item in x if not item.startswith('ditaduranuncamais')])
	# dataframe['Post Created'] = dataframe['Post Created'].dt.tz_convert('UTC')
	dataframe = dataframe[['Post Created', 'image', 'Description', 'Image Text', 'Account', 'User Name'] + [col for col in dataframe.columns if col not in ['Post Created', 'image', 'Description', 'Image Text', 'Account', 'User Name']]]
	return dataframe

	@st.cache_resource(show_spinner=True)
	def load_img_model():
	# We use the original clip-ViT-B-32 for encoding images
	return SentenceTransformer('clip-ViT-B-32')

	@st.cache_resource(show_spinner=True)
	def load_txt_model():
	# Our text embedding model is aligned to the img_model and maps 50+
	# languages to the same vector space
	return SentenceTransformer('./models/sbert.net_models_sentence-transformers_clip-ViT-B-32-multilingual-v1')

	def filter_dataframe(df: pd.DataFrame) -> pd.DataFrame:
	"""
	Adds a UI on top of a dataframe to let viewers filter columns
	Args:
	df (pd.DataFrame): Original dataframe
	Returns:
	pd.DataFrame: Filtered dataframe
	"""
	modify = st.checkbox("Add filters")

	if not modify:
	return df

	df = df.copy()

	# Try to convert datetimes into a standard format (datetime, no timezone)
	for col in df.columns:
	if is_object_dtype(df[col]):
	try:
	df[col] = pd.to_datetime(df[col])
	except Exception:
	pass

	if is_datetime64_any_dtype(df[col]):
	df[col] = df[col].dt.tz_localize(None)

	modification_container = st.container()

	with modification_container:
	to_filter_columns = st.multiselect("Filter dataframe on", df.columns)
	for column in to_filter_columns:
	left, right = st.columns((1, 20))
	left.write("↳")
	# Treat columns with < 10 unique values as categorical
	if is_categorical_dtype(df[column]) or df[column].nunique() < 10:
	user_cat_input = right.multiselect(
	f"Values for {column}",
	df[column].unique(),
	default=list(df[column].unique()),
	)
	df = df[df[column].isin(user_cat_input)]
	elif is_numeric_dtype(df[column]):
	_min = float(df[column].min())
	_max = float(df[column].max())
	step = (_max - _min) / 100
	user_num_input = right.slider(
	f"Values for {column}",
	_min,
	_max,
	(_min, _max),
	step=step,
	)
	df = df[df[column].between(*user_num_input)]
	elif is_datetime64_any_dtype(df[column]):
	user_date_input = right.date_input(
	f"Values for {column}",
	value=(
	df[column].min(),
	df[column].max(),
	),
	)
	if len(user_date_input) == 2:
	user_date_input = tuple(map(pd.to_datetime, user_date_input))
	start_date, end_date = user_date_input
	df = df.loc[df[column].between(start_date, end_date)]
	else:
	user_text_input = right.text_input(
	f"Substring or regex in {column}",
	)
	if user_text_input:
	df = df[df[column].str.contains(user_text_input)]

	return df

	@st.cache_data
	def get_image_embs(image):
	"""
	Get image embeddings
	Parameters:
	uploaded_file (PIL.Image): Uploaded image file
	Returns:
	img_emb (np.array): Image embeddings
	"""
	img_emb = image_model.encode(Image.open(image))
	return img_emb

	@st.cache_data(show_spinner=False)
	def get_text_embs(text):
	"""
	Get text embeddings
	Parameters:
	text (str): Text to encode
	Returns:
	text_emb (np.array): Text embeddings
	"""
	txt_emb = text_model.encode(text)
	return txt_emb

	@st.cache_data
	def postprocess_results(scores, samples):
	"""
	Postprocess results to tuple of labels and scores
	Parameters:
	scores (np.array): Scores
	samples (datasets.Dataset): Samples
	Returns:
	labels (list): List of tuples of PIL images and labels/scores
	"""
	samples_df = pd.DataFrame.from_dict(samples)
	samples_df["score"] = scores
	samples_df["score"] = (1 - (samples_df["score"] - samples_df["score"].min()) / (
	samples_df["score"].max() - samples_df["score"].min())) * 100
	samples_df["score"] = samples_df["score"].astype(int)
	samples_df.reset_index(inplace=True, drop=True)
	samples_df = samples_df[['Post Created', 'image', 'Description', 'Image Text', 'Account', 'User Name'] + [col for col in samples_df.columns if col not in ['Post Created', 'image', 'Description', 'Image Text', 'Account', 'User Name']]]
	return samples_df.drop(columns=['txt_embs', 'img_embs'])

	@st.cache_data
	def text_to_text(text, k=5):
	"""
	Text to text
	Parameters:
	text (str): Input text
	k (int): Number of top results to return
	Returns:
	results (list): List of tuples of PIL images and labels/scores
	"""
	text_emb = get_text_embs(text)
	scores, samples = dataset.get_nearest_examples('txt_embs', text_emb, k=k)
	return postprocess_results(scores, samples)

	@st.cache_data
	def image_to_text(image, k=5):
	"""
	Image to text
	Parameters:
	image (str): Temp filepath to image
	k (int): Number of top results to return
	Returns:
	results (list): List of tuples of PIL images and labels/scores
	"""
	img_emb = get_image_embs(image.name)
	scores, samples = dataset.get_nearest_examples('txt_embs', img_emb, k=k)
	return postprocess_results(scores, samples)

	@st.cache_data
	def text_to_image(text, k=5):
	"""
	Text to image
	Parameters:
	text (str): Input text
	k (int): Number of top results to return
	Returns:
	results (list): List of tuples of PIL images and labels/scores
	"""
	text_emb = get_text_embs(text)
	scores, samples = dataset.get_nearest_examples('img_embs', text_emb, k=k)
	return postprocess_results(scores, samples)

	@st.cache_data
	def image_to_image(image, k=5):
	"""
	Image to image
	Parameters:
	image (str): Temp filepath to image
	k (int): Number of top results to return
	Returns:
	results (list): List of tuples of PIL images and labels/scores
	"""
	img_emb = get_image_embs(image.name)
	scores, samples = dataset.get_nearest_examples('img_embs', img_emb, k=k)
	return postprocess_results(scores, samples)

	def disparity_filter(g: nx.Graph, weight: str = 'weight', alpha: float = 0.05) -> nx.Graph:
	"""
	Computes the backbone of the input graph using the disparity filter algorithm.

	The algorithm is proposed in:
	M. A. Serrano, M. Boguna, and A. Vespignani,
	"Extracting the Multiscale Backbone of Complex Weighted Networks",
	PNAS, 106(16), pp 6483--6488 (2009).
	DOI: 10.1073/pnas.0808904106

	Implementation taken from https://groups.google.com/g/networkx-discuss/c/bCuHZ3qQ2po/m/QvUUJqOYDbIJ

	Parameters
	----------
	g : NetworkX graph
	The input graph.
	weight : str, optional (default='weight')
	The name of the edge attribute to use as weight.
	alpha : float, optional (default=0.05)
	The statistical significance level for the disparity filter (p-value).

	Returns
	-------
	backbone_graph : NetworkX graph
	The backbone graph.
	"""
	# Create an empty graph for the backbone
	backbone_graph = nx.Graph()

	# Iterate over all nodes in the input graph
	for node in g:
	# Get the degree of the node (number of edges connected to the node)
	k_n = len(g[node])

	# Only proceed if the node has more than one connection
	if k_n > 1:
	# Calculate the sum of weights of edges connected to the node
	sum_w = sum(g[node][neighbor][weight] for neighbor in g[node])

	# Iterate over all neighbors of the node
	for neighbor in g[node]:
	# Get the weight of the edge between the node and its neighbor
	edge_weight = g[node][neighbor][weight]

	# Calculate the proportion of the total weight that this edge represents
	pij = float(edge_weight) / sum_w

	# Perform the disparity filter test. If it passes, the edge is considered significant and is added to the backbone
	if (1 - pij) ** (k_n - 1) < alpha:
	backbone_graph.add_edge(node, neighbor, weight=edge_weight)

	# Return the backbone graph
	return backbone_graph

	st.cache_data(show_spinner=True)
	def assign_community_colors(G: nx.Graph, attr: str = 'community') -> dict:
	"""
	Assigns a unique color to each community in the input graph.

	Parameters
	----------
	G : nx.Graph
	The input graph.
	attr : str, optional
	The node attribute of the community names or indexes (default is 'community').

	Returns
	-------
	dict
	A dictionary mapping each community to a unique color.
	"""
	glasbey_colors = cc.glasbey_hv
	communities_ = set(nx.get_node_attributes(G, attr).values())
	return {community: rgb2hex(glasbey_colors[i % len(glasbey_colors)]) for i, community in enumerate(communities_)}

	st.cache_data(show_spinner=True)
	def generate_hover_text(G: nx.Graph, attr: str = 'community') -> list:
	"""
	Generates hover text for each node in the input graph.

	Parameters
	----------
	G : nx.Graph
	The input graph.
	attr : str, optional
	The node attribute of the community names or indexes (default is 'community').

	Returns
	-------
	list
	A list of strings containing the hover text for each node.
	"""
	return [f"Node: {str(node)}<br>Community: {G.nodes[node][attr] + 1}<br># of connections: {len(adjacencies)}" for node, adjacencies in G.adjacency()]

	st.cache_data(show_spinner=True)
	def calculate_node_sizes(G: nx.Graph) -> list:
	"""
	Calculates the size of each node in the input graph based on its degree.

	Parameters
	----------
	G : nx.Graph
	The input graph.

	Returns
	-------
	list
	A list of node sizes.
	"""
	degrees = dict(G.degree())
	max_degree = max(deg for node, deg in degrees.items())
	return [10 + 20 * (degrees[node] / max_degree) for node in G.nodes()]

	@st.cache_data(show_spinner=True)
	def plot_graph(_G: nx.Graph, layout: str = "fdp", community_names_lookup: dict = None):
	"""
	Plots a network graph with communities.

	Parameters
	----------
	G : nx.Graph
	The input graph.
	layout : str, optional
	The layout algorithm to use (default is "fdp").
	"""
	pos = nx.spring_layout(G_backbone, dim=3, seed=779)
	community_colors = assign_community_colors(_G)
	node_colors = [community_colors[_G.nodes[n]['community']] for n in _G.nodes]

	edge_trace = go.Scatter(x=[item for sublist in [[pos[edge[0]][0], pos[edge[1]][0], None] for edge in _G.edges()] for item in sublist],
	y=[item for sublist in [[pos[edge[0]][1], pos[edge[1]][1], None] for edge in _G.edges()] for item in sublist],
	line=dict(width=0.5, color='#888'),
	hoverinfo='none',
	mode='lines')

	node_trace = go.Scatter(x=[pos[n][0] for n in _G.nodes()],
	y=[pos[n][1] for n in _G.nodes()],
	mode='markers',
	hoverinfo='text',
	marker=dict(color=node_colors, size=10, line_width=2))

	node_trace.text = generate_hover_text(_G)
	node_trace.marker.size = calculate_node_sizes(_G)

	fig = go.Figure(data=[edge_trace, node_trace],
	layout=go.Layout(title='Network graph with communities',
	titlefont=dict(size=16),
	showlegend=False,
	hovermode='closest',
	margin=dict(b=20,l=5,r=5,t=40),
	xaxis=dict(showgrid=False, zeroline=False, showticklabels=False),
	yaxis=dict(showgrid=False, zeroline=False, showticklabels=False),
	height=800))

	# Extract node positions
	Xn=[pos[k][0] for k in G_backbone.nodes()] # x-coordinates of nodes
	Yn=[pos[k][1] for k in G_backbone.nodes()] # y-coordinates
	Zn=[pos[k][2] for k in G_backbone.nodes()] # z-coordinates

	# Extract edge positions
	Xe=[]
	Ye=[]
	Ze=[]
	for e in G_backbone.edges():
	Xe+=[pos[e[0]][0],pos[e[1]][0], None] # x-coordinates of edge ends
	Ye+=[pos[e[0]][1],pos[e[1]][1], None]
	Ze+=[pos[e[0]][2],pos[e[1]][2], None]

	# Define traces for plotly
	trace1=go.Scatter3d(x=Xe,
	y=Ye,
	z=Ze,
	mode='lines',
	line=dict(color='rgb(125,125,125)', width=1),
	hoverinfo='none'
	)

	# Map community numbers to names
	community_names = {i: community_names_lookup[f"Community {i+1}"] for i in range(len(communities))}

	# Create hover text
	hover_text = [f"{node} ({community_names[G_backbone.nodes[node]['community']]})" for node in G_backbone.nodes()]

	trace2=go.Scatter3d(x=Xn,
	y=Yn,
	z=Zn,
	mode='markers',
	name='actors',
	marker=dict(symbol='circle',
	size=7,
	color=node_colors, # pass hex colors
	line=dict(color='rgb(50,50,50)', width=0.2)
	),
	text=hover_text, # Use community names as hover text
	hoverinfo='text'
	)

	axis=dict(showbackground=False,
	showline=False,
	zeroline=False,
	showgrid=False,
	showticklabels=False,
	title=''
	)

	layout = go.Layout(
	title="3D Network Graph",
	width=1000,
	height=1000,
	showlegend=False,
	scene=dict(
	xaxis=dict(axis),
	yaxis=dict(axis),
	zaxis=dict(axis),
	),
	margin=dict(
	t=100
	),
	hovermode='closest',
	)

	data=[trace1, trace2]
	fig=go.Figure(data=data, layout=layout)
	return fig

	@st.cache_data(show_spinner=True)
	def cluster_embeddings(embeddings, clustering_algo='KMeans', dim_reduction='PCA', n_clusters=5, min_cluster_size=5, n_components=2, n_neighbors=15, min_dist=0.0, random_state=42, min_samples=5):
	"""
	A function to cluster embeddings.

	Args:
	embeddings (pd.Series): A series of numpy vectors.
	clustering_algo (str): The clustering algorithm to use. Either 'KMeans' or 'HDBSCAN'.
	dim_reduction (str): The dimensionality reduction method to use. Either 'PCA' or 'UMAP'.
	n_clusters (int): The number of clusters for KMeans.
	min_cluster_size (int): The minimum cluster size for HDBSCAN.
	n_components (int): The number of components for the dimensionality reduction method.
	n_neighbors (int): The number of neighbors for UMAP.
	min_dist (float): The minimum distance for UMAP.
	random_state (int): The seed used by the random number generator.
	min_samples (int): The minimum number of samples for HDBSCAN.

	Returns:
	pd.Series: A series of cluster labels.
	"""

	# Dimensionality reduction
	if dim_reduction == 'PCA':
	reducer = PCA(n_components=n_components, random_state=random_state)
	elif dim_reduction == 'UMAP':
	reducer = umap.UMAP(n_neighbors=n_neighbors, min_dist=min_dist, n_components=n_components, random_state=random_state)
	else:
	raise ValueError('Invalid dimensionality reduction method')

	reduced_embeddings = reducer.fit_transform(np.stack(embeddings))

	# Clustering
	if clustering_algo == 'KMeans':
	clusterer = KMeans(n_clusters=n_clusters, random_state=random_state)
	elif clustering_algo == 'HDBSCAN':
	clusterer = hdbscan.HDBSCAN(min_cluster_size=min_cluster_size, min_samples=min_samples)
	else:
	raise ValueError('Invalid clustering algorithm')

	labels = clusterer.fit_predict(reduced_embeddings)

	return labels, reduced_embeddings

	st.title("#ditaduranuncamais Data Explorer")

	def check_password():
	"""Returns `True` if the user had the correct password."""

	def password_entered():
	"""Checks whether a password entered by the user is correct."""
	if st.session_state["password"] == st.secrets["password"]:
	st.session_state["password_correct"] = True
	del st.session_state["password"] # don't store password
	else:
	st.session_state["password_correct"] = False

	if "password_correct" not in st.session_state:
	# First run, show input for password.
	st.text_input(
	"Password", type="password", on_change=password_entered, key="password"
	)
	return False
	elif not st.session_state["password_correct"]:
	# Password not correct, show input + error.
	st.text_input(
	"Password", type="password", on_change=password_entered, key="password"
	)
	st.error("😕 Password incorrect")
	return False
	else:
	# Password correct.
	return True

	if not check_password():
	st.stop()

	# Check if the directory exists
	if not os.path.exists(model_dir):
	download_models()

	dataset = load_dataset()
	df = load_dataframe(dataset)
	image_model = load_img_model()
	text_model = load_txt_model()

	menu_options = ["Data exploration", "Semantic search", "Hashtags", "Clustering", "Stats"]

	st.sidebar.markdown('# Menu')
	selected_menu_option = st.sidebar.radio("Select a page", menu_options)

	if selected_menu_option == "Data exploration":
	st.dataframe(
	data=filter_dataframe(df),
	# use_container_width=True,
	column_config={
	"image": st.column_config.ImageColumn(
	"Image", help="Instagram image"
	),
	"URL": st.column_config.LinkColumn(
	"Link", help="Instagram link", width="small"
	)
	},
	hide_index=True,
	)

	elif selected_menu_option == "Semantic search":
	tabs = ["Text to Text", "Text to Image", "Image to Image", "Image to Text"]
	selected_tab = st.sidebar.radio("Select a search type", tabs)

	if selected_tab == "Text to Text":
	st.markdown('## Text to text search')
	text_to_text_input = st.text_input("Enter text")
	text_to_text_k_top = st.slider("Number of results", 1, 500, 20)
	if st.button("Search"):
	if not text_to_text_input:
	st.warning("Please enter text")
	else:
	st.dataframe(
	data=text_to_text(text_to_text_input, text_to_text_k_top),
	column_config={
	"image": st.column_config.ImageColumn(
	"Image", help="Instagram image"
	),
	"URL": st.column_config.LinkColumn(
	"Link", help="Instagram link", width="small"
	)
	},
	hide_index=True,
	)

	elif selected_tab == "Text to Image":
	st.markdown('## Text to image search')
	text_to_image_input = st.text_input("Enter text")
	text_to_image_k_top = st.slider("Number of results", 1, 500, 20)
	if st.button("Search"):
	if not text_to_image_input:
	st.warning("Please enter some text")
	else:
	st.dataframe(
	data=text_to_image(text_to_image_input, text_to_image_k_top),
	column_config={
	"image": st.column_config.ImageColumn(
	"Image", help="Instagram image"
	),
	"URL": st.column_config.LinkColumn(
	"Link", help="Instagram link", width="small"
	)
	},
	hide_index=True,
	)

	elif selected_tab == "Image to Image":
	st.markdown('## Image to image search')
	image_to_image_k_top = st.slider("Number of results", 1, 500, 20)
	image_to_image_input = st.file_uploader("Upload an image", type=["jpg", "jpeg", "png"])
	temp_file = NamedTemporaryFile(delete=False)
	if st.button("Search"):
	if not image_to_image_input:
	st.warning("Please upload an image")
	else:
	temp_file.write(image_to_image_input.getvalue())

	st.dataframe(
	data=image_to_image(temp_file, image_to_image_k_top),
	column_config={
	"image": st.column_config.ImageColumn(
	"Image", help="Instagram image"
	),
	"URL": st.column_config.LinkColumn(
	"Link", help="Instagram link", width="small"
	)
	},
	hide_index=True,
	)

	elif selected_tab == "Image to Text":
	st.markdown('## Image to text search')
	image_to_text_k_top = st.slider("Number of results", 1, 500, 20)
	image_to_text_input = st.file_uploader("Upload an image", type=["jpg", "jpeg", "png"])
	temp_file = NamedTemporaryFile(delete=False)
	if st.button("Search"):
	if not image_to_text_input:
	st.warning("Please upload an image")
	else:
	temp_file.write(image_to_text_input.getvalue())
	st.dataframe(
	data=image_to_text(temp_file, image_to_text_k_top),
	column_config={
	"image": st.column_config.ImageColumn(
	"Image", help="Instagram image"
	),
	"URL": st.column_config.LinkColumn(
	"Link", help="Instagram link", width="small"
	)
	},
	hide_index=True,
	)
	elif selected_menu_option == "Hashtags":
	if 'dfx' not in st.session_state:
	st.session_state.dfx = df.copy() # Make a copy of dfx
	# Get a list of all unique hashtags in the DataFrame
	all_hashtags = list(set([item for sublist in st.session_state.dfx['Hashtags'].tolist() for item in sublist]))

	st.sidebar.markdown('# Hashtag co-occurrence analysis options')
	# Let users select hashtags to remove
	hashtags_to_remove = st.sidebar.multiselect("Hashtags to remove", all_hashtags)

	col1, col2 = st.sidebar.columns(2)
	# Add a button to trigger the removal operation
	if col1.button("Remove hashtags"):
	# If dfx does not exist in session state, create it
	st.session_state.dfx['Hashtags'] = st.session_state.dfx['Hashtags'].apply(lambda x: [item for item in x if item not in hashtags_to_remove])

	# Add a reset button
	if col2.button("Reset"):
	st.session_state.dfx = df.copy() # Reset dfx to the original DataFrame

	# Count the number of unique hashtags
	hashtags = [item for sublist in st.session_state.dfx['Hashtags'].tolist() for item in sublist]
	# Count the number of posts per hashtag
	hashtag_freq = st.session_state.dfx.explode('Hashtags').groupby('Hashtags').size().reset_index(name='counts')
	# Sort the hashtags by frequency
	hashtag_freq = hashtag_freq.sort_values(by='counts', ascending=False)

	# Make the scatter plot
	hashtags_fig = px.scatter(hashtag_freq, x='Hashtags', y='counts', log_y=True, # Set log_y to True to make the plot more readable on a log scale
	labels={'Hashtags': 'Hashtags', 'counts': 'Frequency'},
	title='Frequency of hashtags in #throwbackthursday posts on Instagram',
	height=600) # Set the height to 600 pixels
	st.markdown("### Hashtag Frequency Distribution")
	st.markdown('Here we apply hashtag co-occurence analysis for mnemonic community detection. This detects communities through creating a network of hashtag pairs (which hashtags are used together in which posts) and then applying community detection algorithms on this network.')
	st.plotly_chart(hashtags_fig)

	weight_option = st.sidebar.radio(
	'Select weight definition',
	('Number of users that use the hashtag pairs', 'Total number of occurrences')
	)

	hashtag_user_pairs = [(tuple(sorted(combination)), userid) for hashtags, userid in zip(st.session_state.dfx['Hashtags'], st.session_state.dfx['User Name']) for combination in combinations(hashtags, r=2)]
	# Create a DataFrame with columns 'hashtag_pair' and 'userid'
	hashtag_user_df = pd.DataFrame(hashtag_user_pairs, columns=['hashtag_pair', 'User Name'])
	if weight_option == 'Number of users that use the hashtag pairs':
	# Group by 'hashtag_pair' and count the number of unique 'userid's
	hashtag_user_df = hashtag_user_df.groupby('hashtag_pair').agg({'User Name': 'nunique'}).reset_index()
	elif weight_option == 'Total number of occurrences':
	# Group by 'hashtag_pair' and count the total number of occurrences
	hashtag_user_df = hashtag_user_df.groupby('hashtag_pair').size().reset_index(name='User Name')
	# Make edge_list from hashtag_user_df with columns 'hashtag1', 'hashtag2', and 'weight'
	edge_list = hashtag_user_df.rename(columns={'hashtag_pair': 'hashtag1', 'User Name': 'weight'})
	edge_list[['hashtag1', 'hashtag2']] = pd.DataFrame(edge_list['hashtag1'].tolist(), index=edge_list.index)
	edge_list = edge_list[['hashtag1', 'hashtag2', 'weight']]

	st.markdown("### Edge List of Hashtag Pairs")
	# Create the graph using the unique users as adge attributes
	G = nx.from_pandas_edgelist(edge_list, 'hashtag1', 'hashtag2', 'weight')
	G_backbone = disparity_filter(G, weight='weight', alpha=0.05)
	st.markdown(f'Number of nodes {len(G_backbone.nodes)}')
	st.markdown(f'Number of edges {len(G_backbone.edges)}')
	st.dataframe(edge_list.sort_values(by='weight', ascending=False).head(10).style.set_caption("Edge list of hashtag pairs with the highest weight"))

	# Create louvain communities
	communities = nx.community.louvain_communities(G_backbone, weight='weight', seed=1234)
	communities = list(communities)

	# Sort communities by size
	communities.sort(key=len, reverse=True)

	for i, community in enumerate(communities):
	for node in community:
	G_backbone.nodes[node]['community'] = i

	# Sort community hashtags based on their weighted degree in the network
	sorted_community_hashtags = [
	[
	hashtag
	for hashtag, degree in sorted(
	((h, G.degree(h, weight='weight')) for h in community),
	key=lambda x: x[1],
	reverse=True
	)
	]
	for community in communities
	]

	# Convert the sorted_community_hashtags list into a DataFrame and transpose it
	sorted_community_hashtags = pd.DataFrame(sorted_community_hashtags).T

	# Rename the columns of sorted_community_hashtags DataFrame
	sorted_community_hashtags.columns = [f'Community {i+1}' for i in range(len(sorted_community_hashtags.columns))]

	st.markdown("### Hashtag Communities")
	st.markdown(f'There are {len(communities)} communities in the graph.')
	st.dataframe(sorted_community_hashtags)

	# add a st.data_editor with Community 1, etc as index and a column "community names" that sets Community 1 etc as default value
	st.markdown("### Community Names")
	st.markdown("Edit the names of the communities in the table below so they show up in the visualisations.")

	df_community_names = pd.DataFrame(sorted_community_hashtags.columns, columns=['community_names'], index=sorted_community_hashtags.columns)
	df_community_names = st.data_editor(df_community_names)
	# download the edited df_community_names as csv
	st.download_button(
	label="Download community names as csv",
	data=df_community_names.to_csv().encode("utf-8"),
	file_name="community_names.csv",
	mime="text/csv",
	)

	#create dict with community names
	community_names_lookup = df_community_names['community_names'].to_dict()

	# implement time series analysis of size of communities over time using resample_dict
	st.markdown("### Community Size Over Time")
	st.markdown("Select communites to see their size over time.")
	# selected_communities = st.multiselect('Select Communities', [f'Community {i+1}' for i in range(len(communities))], default=[f'Community {i+1}' for i in range(len(communities))])
	selected_communities = st.multiselect('Select Communities', community_names_lookup.values(), default=community_names_lookup.values())

	# Dropdown to select time resampling
	resample_dict = {
	'Day': 'D',
	'Three Days': '3D',
	'Week': 'W',
	'Two Weeks': '2W',
	'Month': 'M',
	'Quarter': 'Q',
	'Year': 'Y'
	}

	# Dropdown to select time resampling
	resample_time = st.selectbox('Select Time Resampling', list(resample_dict.keys()), index=4)

	df_communities = st.session_state.dfx.copy()

	def community_dict(communities):
	community_dict = {}
	for i, community in enumerate(communities):
	for node in community:
	community_dict[node] = community_names_lookup[f'Community {i+1}']
	return community_dict

	community_dict = community_dict(communities)

	df_communities['Communities'] = df_communities['Hashtags'].apply(lambda x: [community_dict[tag] for tag in x if tag in community_dict.keys()])

	df_communities = df_communities[['Post Created', 'Communities']].explode('Communities')
	df_communities = df_communities.dropna(subset=['Communities'])

	# Slider for date range selection
	min_date = df_communities['Post Created'].min().date()
	max_date = df_communities['Post Created'].max().date()

	date_range = st.slider('Select Date Range', min_value=min_date, max_value=max_date, value=(min_date, max_date))

	# Filter df_communities by the selected date range
	df_communities = df_communities[(df_communities['Post Created'].dt.date >= date_range[0]) & (df_communities['Post Created'].dt.date <= date_range[1])]

	# Count the number of posts per community per resample_time
	df_communities['Post Created'] = df_communities['Post Created'].dt.to_period(resample_dict[resample_time])
	df_community_sizes = df_communities.groupby(['Post Created', 'Communities']).size().unstack(fill_value=0)
	df_community_sizes.index = df_community_sizes.index.to_timestamp()
	# Filter the DataFrame to include only the selected communities
	df_community_sizes = df_community_sizes[selected_communities]

	st.plotly_chart(px.line(df_community_sizes, title='Community Size Over Time', labels={'value': 'Number of posts', 'index': 'Date', 'variable': 'Community'}))

	st.markdown("### Hashtag Network Graph")
	st.plotly_chart(plot_graph(G_backbone, layout="fdp", community_names_lookup=community_names_lookup)) # fdp is relatively slow, use 'sfdp' or 'neato' for faster but denser layouts


	elif selected_menu_option == "Clustering":
	st.markdown("## Clustering")
	st.markdown("Select the type of embeddings to cluster and the clustering algorithm and dimensionality reduction method to use in the sidebar. Then click run clustering. Clustering may take some time.")
	st.sidebar.markdown("# Clustering Options")
	type_embeddings = st.sidebar.selectbox("Type of embeddings to cluster", ["Text", "Image"])
	clustering_algo = st.sidebar.selectbox("Clustering algorithm", ["HDBSCAN", "KMeans"])
	dim_reduction = st.sidebar.selectbox("Dimensionality reduction method", ["UMAP", "PCA"])
	if clustering_algo == "KMeans":
	st.sidebar.markdown("### KMeans Options")
	n_clusters = st.sidebar.slider("Number of clusters", 2, 20, 5)
	min_cluster_size = None
	min_samples = None
	elif clustering_algo == "HDBSCAN":
	st.sidebar.markdown("### HDBSCAN Options")
	min_cluster_size = st.sidebar.slider("[Minimum cluster size](https://hdbscan.readthedocs.io/en/latest/parameter_selection.html#selecting-min-cluster-size)", 2, 200, 5)
	min_samples = st.sidebar.slider("[Minimum samples](https://hdbscan.readthedocs.io/en/latest/parameter_selection.html#selecting-min-samples)", 2, 50, 5)
	n_clusters = None
	if dim_reduction == "UMAP":
	st.sidebar.markdown("### UMAP Options")
	n_components = st.sidebar.slider("[Number of components](https://umap-learn.readthedocs.io/en/latest/parameters.html#n-components)", 2, 80, 50)
	n_neighbors = st.sidebar.slider("[Number of neighbors](https://umap-learn.readthedocs.io/en/latest/parameters.html#n-neighbors)", 2, 20, 15)
	min_dist = st.sidebar.slider("[Minimum distance](https://umap-learn.readthedocs.io/en/latest/parameters.html#min-dist)", 0.0, 1.0, 0.0)
	else:
	st.sidebar.markdown("### PCA Options")
	n_components = st.sidebar.slider("[Number of components](https://scikit-learn.org/stable/modules/generated/sklearn.decomposition.PCA.html)", 2, 80, 2)
	n_neighbors = None
	min_dist = None

	st.markdown("### Clustering Results")
	if type_embeddings == "Text":
	embeddings = dataset['txt_embs']
	elif type_embeddings == "Image":
	embeddings = dataset['img_embs']

	# Cluster embeddings
	labels, reduced_embeddings = cluster_embeddings(embeddings, clustering_algo=clustering_algo, dim_reduction=dim_reduction, n_clusters=n_clusters, min_cluster_size=min_cluster_size, n_components=n_components, n_neighbors=n_neighbors, min_dist=min_dist)
	st.markdown(f"Clustering {type_embeddings} embeddings using {clustering_algo} with {dim_reduction} dimensionality reduction method resulting in {len(set(labels))} clusters.")

	df_clustered = df.copy()
	df_clustered['cluster'] = labels
	df_clustered = df_clustered.set_index('cluster').reset_index()
	st.dataframe(
	data=filter_dataframe(df_clustered),
	# use_container_width=True,
	column_config={
	"image": st.column_config.ImageColumn(
	"Image", help="Instagram image"
	),
	"URL": st.column_config.LinkColumn(
	"Link", help="Instagram link", width="small"
	)
	},
	hide_index=True,
	)

	st.download_button(
	"Download dataset with labels",
	df_clustered.to_csv(index=False).encode('utf-8'),
	f'ditaduranuncamais_{datetime.now().strftime("%Y%m%d-%H%M%S")}.csv',
	"text/csv",
	key='download-csv'
	)

	st.markdown("### Cluster Plot")
	# Plot the scatter plot in plotly with the cluster labels as colors reduce further to 2 dimensions if n_components > 2
	if n_components > 2:
	reducer = umap.UMAP(n_components=2, random_state=42)
	reduced_embeddings = reducer.fit_transform(reduced_embeddings)
	# set the labels to be the cluster labels dynamically

	# visualise with bokeh showing df_clustered['Description'] and df_clustered['image'] on hover
	descriptions = df_clustered['Description'].tolist()
	images = df_clustered['image'].tolist()
	glasbey_colors = cc.glasbey_hv
	color_dict = {n: rgb2hex(glasbey_colors[i % len(glasbey_colors)]) for i, n in enumerate(set(labels))}
	colors = [color_dict[label] for label in labels]

	source = ColumnDataSource(data=dict(
	x=reduced_embeddings[:, 0],
	y=reduced_embeddings[:, 1],
	desc=descriptions,
	imgs=images,
	colors=colors
	))

	TOOLTIPS = """
	<div>
	<div>
	<img
	src="@imgs" height="100" alt="@imgs" width="100"
	style="float: left; margin: 0px 15px 15px 0px;"
	border="2"
	></img>
	</div>
	<div>
	<span style="font-size: 12px; font-weight: bold;">@desc</span>
	</div>
	</div>
	"""

	p = figure(width=800, height=800, tooltips=TOOLTIPS,
	title="Mouse over the dots")

	p.circle('x', 'y', size=10, source=source, color='colors', line_color=None)
	st.bokeh_chart(p)

	# inster time series graph for clusters sorted by size (except cluster -1, show top5 by default, but include selectbox. reuse resample_dict for binning)
	st.markdown("### Cluster Size")
	cluster_sizes = df_clustered.groupby('cluster').size().reset_index(name='counts')
	cluster_sizes = cluster_sizes.sort_values(by='counts', ascending=False)
	cluster_sizes = cluster_sizes[cluster_sizes['cluster'] != -1]
	cluster_sizes = cluster_sizes.set_index('cluster').reset_index()
	cluster_sizes = cluster_sizes.rename(columns={'cluster': 'Cluster', 'counts': 'Size'})
	st.dataframe(cluster_sizes)

	st.markdown("### Cluster Time Series")

	# Dropdown to select variables
	variable = st.selectbox('Select Variable', ['Likes', 'Comments', 'Followers at Posting', 'Total Interactions'])

	# Dropdown to select time resampling
	resample_dict = {
	'Day': 'D',
	'Three Days': '3D',
	'Week': 'W',
	'Two Weeks': '2W',
	'Month': 'M',
	'Quarter': 'Q',
	'Year': 'Y'
	}

	# Dropdown to select time resampling
	resample_time = st.selectbox('Select Time Resampling', list(resample_dict.keys()))

	# Slider for date range selection
	min_date = df_clustered['Post Created'].min().date()
	max_date = df_clustered['Post Created'].max().date()

	date_range = st.slider('Select Date Range', min_value=min_date, max_value=max_date, value=(min_date, max_date))

	# Filter dataframe based on selected date range
	df_resampled = df_clustered[(df_clustered['Post Created'].dt.date >= date_range[0]) & (df_clustered['Post Created'].dt.date <= date_range[1])]
	df_resampled = df_resampled.set_index('Post Created')

	# Get unique clusters and their sizes
	cluster_sizes = df_resampled[df_resampled['cluster'] != -1]['cluster'].value_counts()
	clusters = cluster_sizes.index

	# Select the largest 5 clusters by default
	default_clusters = cluster_sizes.sort_values(ascending=False).head(5).index.tolist()

	# Multiselect widget to choose clusters
	selected_clusters = st.multiselect('Select Clusters', options=clusters.tolist(), default=default_clusters)

	# Create a new DataFrame for the plot
	df_plot = pd.DataFrame()

	# Loop through selected clusters
	for cluster in selected_clusters:
	# Create a separate DataFrame for each cluster, resample and add to the plot DataFrame
	df_cluster = df_resampled[df_resampled['cluster'] == cluster][variable].resample(resample_dict[resample_time]).sum()
	df_plot = pd.concat([df_plot, df_cluster], axis=1)

	# Add legend (use cluster numbers as legend)
	df_plot.columns = selected_clusters

	# Create the line chart
	st.line_chart(df_plot)



	elif selected_menu_option == "Stats":
	st.markdown("### Time Series Analysis")
	# Dropdown to select variables
	variable = st.selectbox('Select Variable', ['Followers at Posting', 'Total Interactions', 'Likes', 'Comments'])

	# Dropdown to select time resampling
	resample_dict = {
	'Day': 'D',
	'Three Days': '3D',
	'Week': 'W',
	'Two Weeks': '2W',
	'Month': 'M',
	'Quarter': 'Q',
	'Year': 'Y'
	}

	# Dropdown to select time resampling
	resample_time = st.selectbox('Select Time Resampling', list(resample_dict.keys()))

	df_filtered = df.set_index('Post Created')

	# Slider for date range selection
	min_date = df_filtered.index.min().date()
	max_date = df_filtered.index.max().date()

	date_range = st.slider('Select Date Range', min_value=min_date, max_value=max_date, value=(min_date, max_date))

	# Filter dataframe based on selected date range
	df_filtered = df_filtered[(df_filtered.index.date >= date_range[0]) & (df_filtered.index.date <= date_range[1])]

	# Create a separate DataFrame for resampling and plotting
	df_resampled = df_filtered[variable].resample(resample_dict[resample_time]).sum()
	st.line_chart(df_resampled)

	st.markdown("### Correlation Analysis")
	# Dropdown to select variables for scatter plot
	options = ['Followers at Posting', 'Total Interactions', 'Likes', 'Comments']
	scatter_variable_1 = st.selectbox('Select Variable 1 for Scatter Plot', options)
	# options.remove(scatter_variable_1) # remove the chosen option from the list
	scatter_variable_2 = st.selectbox('Select Variable 2 for Scatter Plot', options)

	# Plot scatter chart
	st.write(f"Scatter Plot of {scatter_variable_1} vs {scatter_variable_2}")
	# Plot scatter chart
	scatter_fig = px.scatter(df_filtered, x=scatter_variable_1, y=scatter_variable_2) #, trendline='ols', trendline_color_override='red')

	st.plotly_chart(scatter_fig)

	# calculate correlation for scatter_variable_1 with scatter_variable_2
	corr = df_filtered[scatter_variable_1].corr(df_filtered[scatter_variable_2])
	if corr > 0.7:
	st.write(f"The correlation coefficient is {corr}, indicating a strong positive relationship between {scatter_variable_1} and {scatter_variable_2}.")
	elif corr > 0.3:
	st.write(f"The correlation coefficient is {corr}, indicating a moderate positive relationship between {scatter_variable_1} and {scatter_variable_2}.")
	elif corr > -0.3:
	st.write(f"The correlation coefficient is {corr}, indicating a weak or no relationship between {scatter_variable_1} and {scatter_variable_2}.")
	elif corr > -0.7:
	st.write(f"The correlation coefficient is {corr}, indicating a moderate negative relationship between {scatter_variable_1} and {scatter_variable_2}.")
	else:
	st.write(f"The correlation coefficient is {corr}, indicating a strong negative relationship between {scatter_variable_1} and {scatter_variable_2}.")