pages_clustering.py

from collections import defaultdict
import streamlit as st
from utils import load_and_preprocess_data
import pandas as pd
import numpy as np
import altair as alt
from sklearn.mixture import GaussianMixture
import plotly.express as px
import itertools
from typing import Dict, List, Tuple
from sklearn.preprocessing import LabelEncoder

# Create an instance of LabelEncoder
label_encoder = LabelEncoder()




SIDEBAR_DESCRIPTION = """
# Client clustering

To cluster a client, we adopt the RFM metrics. They stand for:

- R = recency, that is the number of days since the last purchase
    in the store
- F = frequency, that is the number of times a customer has ordered something
- M = monetary value, that is how much a customer has spent buying
    from your business.

Given these 3 metrics, we can cluster the customers and find a suitable
"definition" based on the clusters they belong to. Since the dataset
we're using right now has about 5000 distinct customers, we identify
3 clusters for each metric.

## How we compute the clusters

We resort to a GaussianMixture algorithm. We can think of GaussianMixture
as generalized k-means clustering that incorporates information about
the covariance structure of the data as well as the centers of the clusters.
""".lstrip()

FREQUENCY_CLUSTERS_EXPLAIN = """
The **frequency** denotes how frequently a customer has ordered.

There 3 available clusters for this metric:

- cluster 1: denotes a customer that purchases one or few times (range [{}, {}])
- cluster 2: these customer have a discrete amount of orders (range [{}, {}])
- cluster 3: these customer purchases lots of times (range [{}, {}])

-------
""".lstrip()

RECENCY_CLUSTERS_EXPLAIN = """
The **recency** refers to how recently a customer has bought;

There 3 available clusters for this metric:

- cluster 1: the last order of these client is long time ago (range [{}, {}])
- cluster 2: these are clients that purchases something not very recently (range [{}, {}])
- cluster 3: the last order of these client is a few days/weeks ago (range [{}, {}])

-------
""".lstrip()

MONETARY_CLUSTERS_EXPLAIN = """
The **revenue** refers to how much a customer has spent buying
from your business.

There 3 available clusters for this metric:

- cluster 1: these clients spent little money (range [{}, {}])
- cluster 2: these clients spent a considerable amount of money (range [{}, {}])
- cluster 3: these clients spent lots of money (range [{}, {}])

-------
""".lstrip()

EXPLANATION_DICT = {
    "Frequency_cluster": FREQUENCY_CLUSTERS_EXPLAIN,
    "Recency_cluster": RECENCY_CLUSTERS_EXPLAIN,
    "Revenue_cluster": MONETARY_CLUSTERS_EXPLAIN,
}

# Fit and transform the 'Location' column
merged_df['Location_Encoded'] = label_encoder.fit_transform(merged_df['Location'])
# Assuming 'Age' contains categorical values (e.g., 'young', 'middle-aged', 'old')
merged_df['Age_Encoded'] = label_encoder.fit_transform(merged_df['Age'])

def create_features(df: pd.DataFrame):
    """Creates a new dataframe with the RFM features for each client based on Location and Age."""
    # Compute frequency, the number of distinct books a user has read.
    client_features = df.groupby("User-ID")["ISBN"].nunique().reset_index()
    client_features.columns = ["User-ID", "Frequency"]

    # For this example, let's assume the 'Price' column represents monetary value.
    # Add monetary value, the total revenue for each single user (total books read by the user).
    client_takings = df.groupby("User-ID").size()
    client_features["Total_Books_Read"] = client_takings.values

    # Add recency, let's use the count of unique 'ISBN' as a proxy for recency.
    # You can adjust this based on your specific context.
    client_recency = df.groupby("User-ID")["ISBN"].nunique().reset_index()
    client_recency.columns = ["User-ID", "Recency"]
    client_features["Recency"] = client_recency["Recency"]

    # Incorporating location and age for clustering purposes
    # You might consider encoding location or age if they're categorical
    # For simplicity, assuming both 'Location' and 'Age' are categorical here
    client_location_age = df.drop_duplicates(subset=["User-ID", "Location_Encoded", "Age_Encoded"])
    client_features = client_features.merge(
        client_location_age[["User-ID", "Location", "Age"]],
        on="User-ID",
        how="left",
    )

    return client_features[["User-ID", "Frequency", "Total_Books_Read", "Recency", "Location", "Age"]]



@st.cache
def cluster_clients(df: pd.DataFrame):
    """Computes the RFM features and clusters for each user based on the RFM metrics."""

    df_rfm = create_features(df)

    for to_cluster, order in zip(
        ["Total_Books_Read", "Frequency", "Recency"], ["ascending", "ascending", "descending"]
    ):
        kmeans = GaussianMixture(n_components=3, random_state=42)
        labels = kmeans.fit_predict(df_rfm[[to_cluster]])
        df_rfm[f"{to_cluster}_cluster"] = _order_cluster(kmeans, labels, order)

    return df_rfm


def _order_cluster(cluster_model: GaussianMixture, clusters, order="ascending"):
    """Orders the cluster by `order`."""
    centroids = cluster_model.means_.sum(axis=1)

    if order.lower() == "descending":
        centroids *= -1

    ascending_order = np.argsort(centroids)
    lookup_table = np.zeros_like(ascending_order)
    # Cluster will start from 1
    lookup_table[ascending_order] = np.arange(cluster_model.n_components) + 1
    return lookup_table[clusters]


def show_rating_history(user: int, df: pd.DataFrame):
    user_ratings = df.loc[df["User-ID"] == user, ["Book-Title", "Book-Rating"]]
    
    # Count of books rated by the user
    rated_books_count = user_ratings.shape[0]
    
    st.write(
        f"The user {user} has rated {rated_books_count} books. Here is the rating history:"
    )
    
    st.dataframe(user_ratings)
    
    # Total number of books read by the user
    total_books_read = rated_books_count
    
    st.write(f"Total number of books read by user {user}: {total_books_read}")



def show_user_info(user: int, df_rfm: pd.DataFrame, df_books_read: pd.DataFrame):
    """Prints some information about the user.

    The main information includes age, location,
    total_books_read, and the clusters they belong to.
    """

    user_row = df_rfm[df_rfm["User-ID"] == user]
    if len(user_row) == 0:
        st.write(f"No user with id {user}")

    output = []

    # Fetch user's information from df_rfm
    user_info = user_row.iloc[0]
    output.append(f"Age: {user_info['Age']}")
    output.append(f"Location: {user_info['Location']}")

    # Calculate total_books_read from df_books_read
    total_books_read = df_books_read[df_books_read["User-ID"] == user].shape[0]
    output.append(f"Total books read: {total_books_read}")

    # Display cluster memberships
    output.append("Cluster memberships:")
    for cluster in [column for column in user_row.columns if "_cluster" in column]:
        output.append(f"- {cluster} = {user_row[cluster].squeeze()}")

    st.write("\n".join(output))

    return (
        user_info["Recency_cluster"],
        user_info["Frequency_cluster"],
        user_info["Revenue_cluster"],
    )



def explain_cluster(cluster_info):
    """Displays a popup menu explinging the meanining of the clusters."""

    with st.expander("Show information about the clusters"):
        st.write(
            "**Note**: these values are valid for these dataset."
            "Different dataset will have different number of clusters"
            " and values"
        )
        for cluster, info in cluster_info.items():
            # Transform the (mins, maxs) tuple into
            # [min_1, max_1, min_2, max_2, ...] list.
            min_max_interleaved = list(itertools.chain(*zip(info[0], info[1])))
            st.write(EXPLANATION_DICT[cluster].format(*min_max_interleaved))


def categorize_user(recency_cluster, frequency_cluster, monetary_cluster):
    """Describe the user with few words based on the cluster he belongs to."""

    score = f"{recency_cluster}{frequency_cluster}{monetary_cluster}"

    # @fixme: find a better approeach. These elif chains don't scale at all.

    description = ""

    if score == "111":
        description = "Tourist"
    elif score.startswith("2"):
        description = "Losing interest"
    elif score == "133":
        description = "Former lover"
    elif score == "123":
        description = "Former passionate client"
    elif score == "113":
        description = "Spent a lot, but never come back"
    elif score.startswith("1"):
        description = "About to dump"
    elif score == "313":
        description = "Potential lover"
    elif score == "312":
        description = "Interesting new client"
    elif score == "311":
        description = "New customer"
    elif score == "333":
        description = "Gold client"
    elif score == "322":
        description = "Lovers"
    else:
        description = "Average client"

    st.write(f"The customer can be described as: **{description}**")


def plot_rfm_distribution(
    df_rfm: pd.DataFrame, cluster_info: Dict[str, Tuple[List[int], List[int]]]
):
    """Plots 3 histograms for the RFM metrics."""

    for x, to_reverse in zip(("Revenue", "Frequency", "Recency"), (False, False, True)):
        fig = px.histogram(
            df_rfm,
            x=x,
            log_y=True,
            title=f"{x} metric",
        )
        # Get the max value in the cluster info. The cluster_info_dict is a
        # tuple with first element the min values of the cluster, and second
        # element the max values of the cluster.
        values = cluster_info[f"{x}_cluster"][1]  # get max values
        print(values)
        # Add vertical bar on each cluster end. But skip the last cluster.
        loop_range = range(len(values) - 1)
        if to_reverse:
            # Skip the last element
            loop_range = range(len(values) - 1, 0, -1)
        for n_cluster in loop_range:
            print(x)
            print(values[n_cluster])
            fig.add_vline(
                x=values[n_cluster],
                annotation_text=f"End of cluster {n_cluster+1}",
                line_dash="dot",
                annotation=dict(textangle=90, font_color="red"),
            )

        fig.update_layout(
            yaxis_title="Count (log scale)",
        )

        st.plotly_chart(fig)


def display_dataframe_heatmap(df_rfm: pd.DataFrame, cluster_info_dict):
    """Displays an heatmap of how many clients lay in the clusters.

    This method uses some black magic coming from the dataframe
    styling guide.
    """

    def style_with_limits(x, column, cluster_limit_dict):
        """Simple function to transform the cluster number into
        a cluster + range string."""
        min_v = cluster_limit_dict[column][0][x - 1]
        max_v = cluster_limit_dict[column][1][x - 1]
        return f"{x}: [{int(min_v)}, {int(max_v)}]"

    # Create a dataframe with the count of clients for each group
    # of cluster.

    count = (
        df_rfm.groupby(["Recency_cluster", "Frequency_cluster", "Revenue_cluster"])[
            "User-ID"
        ]
        .count()
        .reset_index()
    )
    count = count.rename(columns={"User-ID": "Count"})

    # Remove duplicates
    count = count.drop_duplicates(
        ["Revenue_cluster", "Frequency_cluster", "Recency_cluster"]
    )

    # Add limits to the cells. In this way, we can better display
    # the heatmap.
    for cluster in ["Revenue_cluster", "Frequency_cluster", "Recency_cluster"]:
        count[cluster] = count[cluster].apply(
            lambda x: style_with_limits(x, cluster, cluster_info_dict)
        )

    # Use the count column as values, then index with the clusters.
    count = count.pivot(
        index=["Revenue_cluster", "Frequency_cluster"],
        columns="Recency_cluster",
        values="Count",
    )

    # Style manipulation
    cell_hover = {
        "selector": "td",
        "props": "font-size:1.2em",
    }
    index_names = {
        "selector": ".index_name",
        "props": "font-style: italic; color: Black; font-weight:normal;font-size:1.2em;",
    }
    headers = {
        "selector": "th:not(.index_name)",
        "props": "background-color: White; color: black; font-size:1.2em",
    }

    # Finally, display
    # We cannot directly print the dataframe since the streamlit
    # functin remove the multiindex. Thus, we extract the html representation
    # and then display it.
    st.markdown("## Heatmap: how the client are distributed between clusters")
    st.write(
        count.style.format(thousands=" ", precision=0, na_rep="0")
        .set_table_styles([cell_hover, index_names, headers])
        .background_gradient(cmap="coolwarm")
        .to_html(),
        unsafe_allow_html=True,
    )


def main():
    st.sidebar.markdown(SIDEBAR_DESCRIPTION)

    df, _, _ = load_and_preprocess_data()
    df_rfm = cluster_clients(df)

    st.markdown(
        "# Dataset "
        "\nThis is the processed dataset with information about the clients, such as"
        " the RFM values and the clusters they belong to."
    )
    st.dataframe(df_rfm.style.format(formatter={"Revenue": "{:.2f}"}))

    cluster_info_dict = defaultdict(list)

    with st.expander("Show more details about the clusters"):
        for cluster in [column for column in df_rfm.columns if "_cluster" in column]:
            st.write(cluster)
            cluster_info = (
                df_rfm.groupby(cluster)[cluster.split("_")[0]]
                .describe()
                .reset_index(names="Cluster")
            )
            min_cluster = cluster_info["min"].astype(int)
            max_cluster = cluster_info["max"].astype(int)
            cluster_info_dict[cluster] = (min_cluster, max_cluster)
            st.dataframe(cluster_info)

    st.markdown("## RFM metric distribution")

    plot_rfm_distribution(df_rfm, cluster_info_dict)

    display_dataframe_heatmap(df_rfm, cluster_info_dict)

    st.markdown("## Interactive exploration")

    filter_by_cluster = st.checkbox(
        "Filter client: only one client per cluster type",
        value=True,
    )

    client_to_select = (
        df_rfm.groupby(["Recency_cluster", "Frequency_cluster", "Revenue_cluster"])[
            "User-ID"
        ]
        .first()
        .values
        if filter_by_cluster
        else df["User-ID"].unique()
    )

    # Let the user select the user to investigate
    user = st.selectbox(
        "Select a customer to show more information about him.",
        client_to_select,
    )

    show_purhcase_history(user, df)

    recency, frequency, revenue = show_user_info(user, df_rfm)

    categorize_user(recency, frequency, revenue)

    explain_cluster(cluster_info_dict)


main()