Spaces:
Sleeping
Sleeping
| from collections import defaultdict | |
| import streamlit as st | |
| from utils import load_and_preprocess_data | |
| import pandas as pd | |
| import numpy as np | |
| import altair as alt | |
| from sklearn.mixture import GaussianMixture | |
| import plotly.express as px | |
| import itertools | |
| from typing import Dict, List, Tuple | |
| from sklearn.preprocessing import LabelEncoder | |
| # Create an instance of LabelEncoder | |
| label_encoder = LabelEncoder() | |
| SIDEBAR_DESCRIPTION = """ | |
| # Client clustering | |
| To cluster a client, we adopt the RFM metrics. They stand for: | |
| - R = recency, that is the number of days since the last purchase | |
| in the store | |
| - F = frequency, that is the number of times a customer has ordered something | |
| - M = monetary value, that is how much a customer has spent buying | |
| from your business. | |
| Given these 3 metrics, we can cluster the customers and find a suitable | |
| "definition" based on the clusters they belong to. Since the dataset | |
| we're using right now has about 5000 distinct customers, we identify | |
| 3 clusters for each metric. | |
| ## How we compute the clusters | |
| We resort to a GaussianMixture algorithm. We can think of GaussianMixture | |
| as generalized k-means clustering that incorporates information about | |
| the covariance structure of the data as well as the centers of the clusters. | |
| """.lstrip() | |
| FREQUENCY_CLUSTERS_EXPLAIN = """ | |
| The **frequency** denotes how frequently a customer has ordered. | |
| There 3 available clusters for this metric: | |
| - cluster 1: denotes a customer that purchases one or few times (range [{}, {}]) | |
| - cluster 2: these customer have a discrete amount of orders (range [{}, {}]) | |
| - cluster 3: these customer purchases lots of times (range [{}, {}]) | |
| ------- | |
| """.lstrip() | |
| RECENCY_CLUSTERS_EXPLAIN = """ | |
| The **recency** refers to how recently a customer has bought; | |
| There 3 available clusters for this metric: | |
| - cluster 1: the last order of these client is long time ago (range [{}, {}]) | |
| - cluster 2: these are clients that purchases something not very recently (range [{}, {}]) | |
| - cluster 3: the last order of these client is a few days/weeks ago (range [{}, {}]) | |
| ------- | |
| """.lstrip() | |
| MONETARY_CLUSTERS_EXPLAIN = """ | |
| The **revenue** refers to how much a customer has spent buying | |
| from your business. | |
| There 3 available clusters for this metric: | |
| - cluster 1: these clients spent little money (range [{}, {}]) | |
| - cluster 2: these clients spent a considerable amount of money (range [{}, {}]) | |
| - cluster 3: these clients spent lots of money (range [{}, {}]) | |
| ------- | |
| """.lstrip() | |
| EXPLANATION_DICT = { | |
| "Frequency_cluster": FREQUENCY_CLUSTERS_EXPLAIN, | |
| "Recency_cluster": RECENCY_CLUSTERS_EXPLAIN, | |
| "Revenue_cluster": MONETARY_CLUSTERS_EXPLAIN, | |
| } | |
| # Fit and transform the 'Location' column | |
| merged_df['Location_Encoded'] = label_encoder.fit_transform(merged_df['Location']) | |
| # Assuming 'Age' contains categorical values (e.g., 'young', 'middle-aged', 'old') | |
| merged_df['Age_Encoded'] = label_encoder.fit_transform(merged_df['Age']) | |
| def create_features(df: pd.DataFrame): | |
| """Creates a new dataframe with the RFM features for each client based on Location and Age.""" | |
| # Compute frequency, the number of distinct books a user has read. | |
| client_features = df.groupby("User-ID")["ISBN"].nunique().reset_index() | |
| client_features.columns = ["User-ID", "Frequency"] | |
| # For this example, let's assume the 'Price' column represents monetary value. | |
| # Add monetary value, the total revenue for each single user (total books read by the user). | |
| client_takings = df.groupby("User-ID").size() | |
| client_features["Total_Books_Read"] = client_takings.values | |
| # Add recency, let's use the count of unique 'ISBN' as a proxy for recency. | |
| # You can adjust this based on your specific context. | |
| client_recency = df.groupby("User-ID")["ISBN"].nunique().reset_index() | |
| client_recency.columns = ["User-ID", "Recency"] | |
| client_features["Recency"] = client_recency["Recency"] | |
| # Incorporating location and age for clustering purposes | |
| # You might consider encoding location or age if they're categorical | |
| # For simplicity, assuming both 'Location' and 'Age' are categorical here | |
| client_location_age = df.drop_duplicates(subset=["User-ID", "Location_Encoded", "Age_Encoded"]) | |
| client_features = client_features.merge( | |
| client_location_age[["User-ID", "Location", "Age"]], | |
| on="User-ID", | |
| how="left", | |
| ) | |
| return client_features[["User-ID", "Frequency", "Total_Books_Read", "Recency", "Location", "Age"]] | |
| def cluster_clients(df: pd.DataFrame): | |
| """Computes the RFM features and clusters for each user based on the RFM metrics.""" | |
| df_rfm = create_features(df) | |
| for to_cluster, order in zip( | |
| ["Total_Books_Read", "Frequency", "Recency"], ["ascending", "ascending", "descending"] | |
| ): | |
| kmeans = GaussianMixture(n_components=3, random_state=42) | |
| labels = kmeans.fit_predict(df_rfm[[to_cluster]]) | |
| df_rfm[f"{to_cluster}_cluster"] = _order_cluster(kmeans, labels, order) | |
| return df_rfm | |
| def _order_cluster(cluster_model: GaussianMixture, clusters, order="ascending"): | |
| """Orders the cluster by `order`.""" | |
| centroids = cluster_model.means_.sum(axis=1) | |
| if order.lower() == "descending": | |
| centroids *= -1 | |
| ascending_order = np.argsort(centroids) | |
| lookup_table = np.zeros_like(ascending_order) | |
| # Cluster will start from 1 | |
| lookup_table[ascending_order] = np.arange(cluster_model.n_components) + 1 | |
| return lookup_table[clusters] | |
| def show_rating_history(user: int, df: pd.DataFrame): | |
| user_ratings = df.loc[df["User-ID"] == user, ["Book-Title", "Book-Rating"]] | |
| # Count of books rated by the user | |
| rated_books_count = user_ratings.shape[0] | |
| st.write( | |
| f"The user {user} has rated {rated_books_count} books. Here is the rating history:" | |
| ) | |
| st.dataframe(user_ratings) | |
| # Total number of books read by the user | |
| total_books_read = rated_books_count | |
| st.write(f"Total number of books read by user {user}: {total_books_read}") | |
| def show_user_info(user: int, df_rfm: pd.DataFrame, df_books_read: pd.DataFrame): | |
| """Prints some information about the user. | |
| The main information includes age, location, | |
| total_books_read, and the clusters they belong to. | |
| """ | |
| user_row = df_rfm[df_rfm["User-ID"] == user] | |
| if len(user_row) == 0: | |
| st.write(f"No user with id {user}") | |
| output = [] | |
| # Fetch user's information from df_rfm | |
| user_info = user_row.iloc[0] | |
| output.append(f"Age: {user_info['Age']}") | |
| output.append(f"Location: {user_info['Location']}") | |
| # Calculate total_books_read from df_books_read | |
| total_books_read = df_books_read[df_books_read["User-ID"] == user].shape[0] | |
| output.append(f"Total books read: {total_books_read}") | |
| # Display cluster memberships | |
| output.append("Cluster memberships:") | |
| for cluster in [column for column in user_row.columns if "_cluster" in column]: | |
| output.append(f"- {cluster} = {user_row[cluster].squeeze()}") | |
| st.write("\n".join(output)) | |
| return ( | |
| user_info["Recency_cluster"], | |
| user_info["Frequency_cluster"], | |
| user_info["Revenue_cluster"], | |
| ) | |
| def explain_cluster(cluster_info): | |
| """Displays a popup menu explinging the meanining of the clusters.""" | |
| with st.expander("Show information about the clusters"): | |
| st.write( | |
| "**Note**: these values are valid for these dataset." | |
| "Different dataset will have different number of clusters" | |
| " and values" | |
| ) | |
| for cluster, info in cluster_info.items(): | |
| # Transform the (mins, maxs) tuple into | |
| # [min_1, max_1, min_2, max_2, ...] list. | |
| min_max_interleaved = list(itertools.chain(*zip(info[0], info[1]))) | |
| st.write(EXPLANATION_DICT[cluster].format(*min_max_interleaved)) | |
| def categorize_user(recency_cluster, frequency_cluster, monetary_cluster): | |
| """Describe the user with few words based on the cluster he belongs to.""" | |
| score = f"{recency_cluster}{frequency_cluster}{monetary_cluster}" | |
| # @fixme: find a better approeach. These elif chains don't scale at all. | |
| description = "" | |
| if score == "111": | |
| description = "Tourist" | |
| elif score.startswith("2"): | |
| description = "Losing interest" | |
| elif score == "133": | |
| description = "Former lover" | |
| elif score == "123": | |
| description = "Former passionate client" | |
| elif score == "113": | |
| description = "Spent a lot, but never come back" | |
| elif score.startswith("1"): | |
| description = "About to dump" | |
| elif score == "313": | |
| description = "Potential lover" | |
| elif score == "312": | |
| description = "Interesting new client" | |
| elif score == "311": | |
| description = "New customer" | |
| elif score == "333": | |
| description = "Gold client" | |
| elif score == "322": | |
| description = "Lovers" | |
| else: | |
| description = "Average client" | |
| st.write(f"The customer can be described as: **{description}**") | |
| def plot_rfm_distribution( | |
| df_rfm: pd.DataFrame, cluster_info: Dict[str, Tuple[List[int], List[int]]] | |
| ): | |
| """Plots 3 histograms for the RFM metrics.""" | |
| for x, to_reverse in zip(("Revenue", "Frequency", "Recency"), (False, False, True)): | |
| fig = px.histogram( | |
| df_rfm, | |
| x=x, | |
| log_y=True, | |
| title=f"{x} metric", | |
| ) | |
| # Get the max value in the cluster info. The cluster_info_dict is a | |
| # tuple with first element the min values of the cluster, and second | |
| # element the max values of the cluster. | |
| values = cluster_info[f"{x}_cluster"][1] # get max values | |
| print(values) | |
| # Add vertical bar on each cluster end. But skip the last cluster. | |
| loop_range = range(len(values) - 1) | |
| if to_reverse: | |
| # Skip the last element | |
| loop_range = range(len(values) - 1, 0, -1) | |
| for n_cluster in loop_range: | |
| print(x) | |
| print(values[n_cluster]) | |
| fig.add_vline( | |
| x=values[n_cluster], | |
| annotation_text=f"End of cluster {n_cluster+1}", | |
| line_dash="dot", | |
| annotation=dict(textangle=90, font_color="red"), | |
| ) | |
| fig.update_layout( | |
| yaxis_title="Count (log scale)", | |
| ) | |
| st.plotly_chart(fig) | |
| def display_dataframe_heatmap(df_rfm: pd.DataFrame, cluster_info_dict): | |
| """Displays an heatmap of how many clients lay in the clusters. | |
| This method uses some black magic coming from the dataframe | |
| styling guide. | |
| """ | |
| def style_with_limits(x, column, cluster_limit_dict): | |
| """Simple function to transform the cluster number into | |
| a cluster + range string.""" | |
| min_v = cluster_limit_dict[column][0][x - 1] | |
| max_v = cluster_limit_dict[column][1][x - 1] | |
| return f"{x}: [{int(min_v)}, {int(max_v)}]" | |
| # Create a dataframe with the count of clients for each group | |
| # of cluster. | |
| count = ( | |
| df_rfm.groupby(["Recency_cluster", "Frequency_cluster", "Revenue_cluster"])[ | |
| "User-ID" | |
| ] | |
| .count() | |
| .reset_index() | |
| ) | |
| count = count.rename(columns={"User-ID": "Count"}) | |
| # Remove duplicates | |
| count = count.drop_duplicates( | |
| ["Revenue_cluster", "Frequency_cluster", "Recency_cluster"] | |
| ) | |
| # Add limits to the cells. In this way, we can better display | |
| # the heatmap. | |
| for cluster in ["Revenue_cluster", "Frequency_cluster", "Recency_cluster"]: | |
| count[cluster] = count[cluster].apply( | |
| lambda x: style_with_limits(x, cluster, cluster_info_dict) | |
| ) | |
| # Use the count column as values, then index with the clusters. | |
| count = count.pivot( | |
| index=["Revenue_cluster", "Frequency_cluster"], | |
| columns="Recency_cluster", | |
| values="Count", | |
| ) | |
| # Style manipulation | |
| cell_hover = { | |
| "selector": "td", | |
| "props": "font-size:1.2em", | |
| } | |
| index_names = { | |
| "selector": ".index_name", | |
| "props": "font-style: italic; color: Black; font-weight:normal;font-size:1.2em;", | |
| } | |
| headers = { | |
| "selector": "th:not(.index_name)", | |
| "props": "background-color: White; color: black; font-size:1.2em", | |
| } | |
| # Finally, display | |
| # We cannot directly print the dataframe since the streamlit | |
| # functin remove the multiindex. Thus, we extract the html representation | |
| # and then display it. | |
| st.markdown("## Heatmap: how the client are distributed between clusters") | |
| st.write( | |
| count.style.format(thousands=" ", precision=0, na_rep="0") | |
| .set_table_styles([cell_hover, index_names, headers]) | |
| .background_gradient(cmap="coolwarm") | |
| .to_html(), | |
| unsafe_allow_html=True, | |
| ) | |
| def main(): | |
| st.sidebar.markdown(SIDEBAR_DESCRIPTION) | |
| df, _, _ = load_and_preprocess_data() | |
| df_rfm = cluster_clients(df) | |
| st.markdown( | |
| "# Dataset " | |
| "\nThis is the processed dataset with information about the clients, such as" | |
| " the RFM values and the clusters they belong to." | |
| ) | |
| st.dataframe(df_rfm.style.format(formatter={"Revenue": "{:.2f}"})) | |
| cluster_info_dict = defaultdict(list) | |
| with st.expander("Show more details about the clusters"): | |
| for cluster in [column for column in df_rfm.columns if "_cluster" in column]: | |
| st.write(cluster) | |
| cluster_info = ( | |
| df_rfm.groupby(cluster)[cluster.split("_")[0]] | |
| .describe() | |
| .reset_index(names="Cluster") | |
| ) | |
| min_cluster = cluster_info["min"].astype(int) | |
| max_cluster = cluster_info["max"].astype(int) | |
| cluster_info_dict[cluster] = (min_cluster, max_cluster) | |
| st.dataframe(cluster_info) | |
| st.markdown("## RFM metric distribution") | |
| plot_rfm_distribution(df_rfm, cluster_info_dict) | |
| display_dataframe_heatmap(df_rfm, cluster_info_dict) | |
| st.markdown("## Interactive exploration") | |
| filter_by_cluster = st.checkbox( | |
| "Filter client: only one client per cluster type", | |
| value=True, | |
| ) | |
| client_to_select = ( | |
| df_rfm.groupby(["Recency_cluster", "Frequency_cluster", "Revenue_cluster"])[ | |
| "User-ID" | |
| ] | |
| .first() | |
| .values | |
| if filter_by_cluster | |
| else df["User-ID"].unique() | |
| ) | |
| # Let the user select the user to investigate | |
| user = st.selectbox( | |
| "Select a customer to show more information about him.", | |
| client_to_select, | |
| ) | |
| show_purhcase_history(user, df) | |
| recency, frequency, revenue = show_user_info(user, df_rfm) | |
| categorize_user(recency, frequency, revenue) | |
| explain_cluster(cluster_info_dict) | |
| main() | |