Spaces:
Sleeping
Sleeping
from data_utils.data_generation import UpliftSimulation | |
from data_utils.exploratory_data_analysis import ExploratoryAnalysis | |
from data_utils.feature_importance import FeatureImportance | |
from models_utils.ml_models import ModelTraining | |
from eval_utils.evaluation import ModelEvaluator | |
import matplotlib.pyplot as plt | |
import plotly.express as px | |
import plotly.graph_objects as go | |
import numpy as np | |
import pandas as pd | |
import streamlit as st | |
X_names = [ | |
'AgeIndex', 'IncomeIndex', 'PurchaseFrequencyIndex', | |
'AccountLifetimeIndex', 'AverageTransactionValueIndex', 'PreferredPaymentMethodIndex', 'RegionIndex', | |
'EmailDiscountCTRIndex', 'WebDiscountCTRIndex', 'SocialMediaEngagementIndex', | |
'DirectMailDiscountResponseIndex', 'InAppDiscountEngagementIndex', 'FlashSaleParticipationIndex', | |
'SeasonalPromoInterestIndex', 'LoyaltyProgramEngagementIndex', 'ReferralBonusUsageIndex', | |
'DiscountCodeRedemptionIndex', 'VIPSaleAccessIndex', 'EarlyAccessOptInIndex', | |
'ProductReviewAfterDiscountIndex', 'UpsellConversionIndex', 'CrossSellInterestIndex', | |
'BundlePurchaseIndex', 'SubscriptionUpgradeIndex', 'CustomerFeedbackIndex', | |
'BrowserTypeIndex', 'DeviceCategoryIndex', 'OperatingSystemIndex', | |
'SessionStartTimeIndex', 'LanguagePreferenceIndex', 'NewsletterSubscriptionIndex', | |
'AccountVerificationStatusIndex', 'AdBlockerPresenceIndex' | |
] | |
# Title | |
st.title("Uplift Modeling in Retail Demo") | |
tabs = st.sidebar.radio("Navigation", [ | |
"Overview", | |
"Data generation", | |
"Exploratory analysis", | |
"Model training", | |
"Economic effects"] | |
) | |
if tabs == "Overview": | |
st.header("Overview") | |
st.write(""" | |
This app demonstrates the use of uplift modeling to understand the effect of different actions (like promotions) on customer behavior. We generate a simulated dataset and use it to train a model that predicts the uplift effect of different treatments on customer behavior. We then evaluate the model using the Qini curve, which measures the uplift effect of a model compared to a random model. | |
""") | |
VIDEO_URL = "https://www.youtube.com/watch?v=AIL0QZWSWBY" | |
st.video(VIDEO_URL) | |
st.write("To get started, select the 'Data generation' tab from the sidebar.") | |
st.write("For more details, reach out to info@neurons-lab.com") | |
if tabs == "Data generation": | |
st.header("Data Generation") | |
# Description | |
st.write(""" | |
This app creates a simulated dataset for a special kind of analysis called uplift modeling, which helps understand the effect of different actions (like promotions) on customer behavior. We use some default settings to make things easy: | |
- We're looking at whether customers make a purchase or not. | |
- We compare different types of promotions (like no discount, 5% off, etc.). | |
- The dataset includes 15 different pieces of information (features) about each customer. | |
""") | |
# Interactive number of samples selection | |
n = st.number_input('Number of Samples (n)', min_value=1000, value=10000, step=1000, | |
help="Total number of samples to generate in the dataset.") | |
# Default values for other variables | |
y_name = 'conversion' | |
treatment_group_keys = ['control', 'discount_05', 'discount_10', 'discount_15'] | |
n_classification_features = 15 | |
n_classification_informative = 7 | |
n_classification_repeated = 0 | |
n_uplift_increase_dict = {'discount_05': 4, 'discount_10': 3, 'discount_15': 3} | |
n_uplift_decrease_dict = {'discount_05': 0, 'discount_10': 0, 'discount_15': 0} | |
positive_class_proportion = 0.05 | |
random_seed = 8097 | |
# Button to generate dataset | |
if st.button('Generate Dataset'): | |
uplift_sim = UpliftSimulation(n=n, y_name=y_name, treatment_group_keys=treatment_group_keys, | |
n_classification_features=n_classification_features, | |
n_classification_informative=n_classification_informative, | |
n_classification_repeated=n_classification_repeated, | |
n_uplift_increase_dict=n_uplift_increase_dict, | |
n_uplift_decrease_dict=n_uplift_decrease_dict, | |
positive_class_proportion=positive_class_proportion, | |
random_seed=random_seed) | |
uplift_sim.simulate_dataset() | |
uplift_sim.apply_discounts_and_clean() | |
uplift_sim.postprocess_tables() | |
uplift_sim.add_monetary_effect() | |
st.session_state.uplift_sim = uplift_sim # Store in session state | |
st.write("Dataset Generated Successfully!") | |
st.subheader("User profiles") | |
st.write('Features that represent a customer such as age, income, purchase frequency, etc') | |
st.dataframe(uplift_sim.dataframes[0].head(3)) | |
st.subheader("Treatments data") | |
st.write('Information about the different treatments (discounts) that were applied to the customers as discounts in different channels (web, email, mobile), early access, etc') | |
st.dataframe(uplift_sim.dataframes[1].head(3)) | |
st.subheader("Other data") | |
st.write('Other data that can be used in the analysis') | |
st.dataframe(uplift_sim.dataframes[2].head(3)) | |
if tabs == "Exploratory analysis": | |
st.header("Exploratory Analysis") | |
if 'uplift_sim' in st.session_state: | |
st.subheader('Summary statistics') | |
uplift_sim = st.session_state.uplift_sim | |
eda = ExploratoryAnalysis(uplift_sim.df) | |
st.write('We begin by computing the total sum of conversions, sales (discounted price) and platform benefit. We can see that the total conversions and the total sales grows as the discount value is bigger. However, the platform benefit decreases.') | |
sum_conversions, mean_conversions = eda.compute_summaries() | |
st.write(sum_conversions) | |
st.write(mean_conversions) | |
st.write('We can also visualize the tradeoff between conversions and platform benefit by plotting the mean benefit per user on the y-axis and the mean conversion rate on the x-axis, for each treatment group.') | |
mean_benefit_vs_conversion = eda.compute_mean_benefit_vs_conversion() | |
# fig, ax = plt.subplots() | |
# mean_benefit_vs_conversion.plot.scatter(x='conversion', y='benefit', c='DarkBlue', s=50, ax=ax) | |
# st.pyplot(fig) | |
fig = px.scatter(mean_benefit_vs_conversion, x='conversion', y='benefit', color_discrete_sequence=['LightBlue'], size_max=50) | |
st.plotly_chart(fig) | |
st.write(''' | |
We further compute the Average Treatment Effect (ATE) for both the mean conversion rate and the mean benefit per user: | |
- Conversion ATE = Mean Conversion rate in the discounted group minus Mean Conversion rate in the control group | |
- Benefit ATE = Mean Benefit per user in the discounted group minus Mean Benefit per user in the control group | |
This helps illustrate how the discount value affects Conversion ATE and Benefit ATE. | |
''') | |
mean_conversions_ate = eda.compute_ate() | |
# fig, ax = plt.subplots() | |
# mean_conversions_ate.plot.scatter(x='conversion', y='benefit', c='DarkBlue', s=50, ax=ax) | |
# st.pyplot(fig) | |
fig = px.scatter(mean_conversions_ate, x='conversion', y='benefit', color_discrete_sequence=['LightBlue'], size_max=50) | |
st.plotly_chart(fig) | |
st.subheader('Feature importance') | |
# Allow users to select a treatment group | |
treatment_group = st.selectbox( | |
'Select a treatment group', | |
options=['discount_05', 'discount_10', 'discount_15'], | |
index=0 # default to 'discount_05' | |
) | |
feature_importance = FeatureImportance(uplift_sim.df, X_names, y_name = 'conversion', treatment_group = treatment_group) | |
fi = feature_importance.compute_feature_importance() | |
# fig, ax = plt.subplots() | |
di_df_sorted = fi.sort_values(by='score', ascending=False) | |
# di_df_sorted[['feature', 'score']].plot.barh(x='feature', y='score', ax=ax) | |
# st.pyplot(fig) | |
fig = px.bar(di_df_sorted, y='feature', x='score', orientation='h') | |
st.plotly_chart(fig) | |
st.write(""" | |
- AccountLifetimeIndex: Longer-standing accounts are key predictors of customer response to promotions \n | |
- CustomerFeedbackIndex: Customer feedback significantly influences the success of marketing strategies \n | |
- UpsellConversionIndex: The success rate of upselling is an important factor \n | |
- PurchaseFrequencyIndex: More frequent purchases indicate higher engagement and response to marketing efforts \n | |
- ReferralBonusUsedIndex and LoyaltyProgramEngagementIndex: Engagement with these programs is highly indicative of responsiveness to promotions | |
""") | |
else: | |
st.error("Please generate the dataset first.") | |
if tabs == "Model training": | |
st.header("Model Training") | |
st.write(""" | |
In this section, we train a model to predict the uplift effect of different treatments on customer behavior. | |
We use the XGBoost algorithm to train the model. The model can be used to predict the conversion rate or the benefit per user for each treatment group. | |
We can also analyze the economic effects of the treatments by comparing the uplift in conversion rate and benefit per user. | |
""") | |
if 'uplift_sim' in st.session_state: | |
uplift_sim = st.session_state.uplift_sim | |
model_trainer = ModelTraining(uplift_sim.df, 'conversion', X_names) | |
model_type = st.radio("Choose the model type", ('Conversion Model', 'Benefit Model')) | |
params = { | |
'n_estimators': st.slider('Number of Estimators', 10, 100, 50), | |
'max_depth': st.slider('Max Depth', 1, 10, 4), | |
'colsample_bytree': st.slider('Colsample by Tree', 0.1, 1.0, 0.2), | |
'subsample': st.slider('Subsample', 0.1, 1.0, 0.2), | |
} | |
control_name = 'control' # st.text_input('Control Group Name', 'control') | |
test_size = st.slider('Test Size', 0.1, 0.9, 0.5) | |
random_state = 20143 # st.slider('Random State', 0, 10000, 20143) | |
if st.button('Train Model'): | |
model_trainer.split_data(test_size=test_size, random_state=random_state) | |
if model_type == 'Conversion Model': | |
y_name = 'conversion' # st.selectbox('Select target variable for conversion', options=uplift_sim.target_options) | |
model_trainer.y_name = y_name | |
tau = model_trainer.fit_predict_classifier(params, control_name) | |
elif model_type == 'Benefit Model': | |
y_name = 'benefit' # st.selectbox('Select target variable for benefit', options=uplift_sim.benefit_options) | |
model_trainer.y_name = y_name | |
tau = model_trainer.fit_predict_regressor(params, control_name) | |
st.session_state.model_trainer = model_trainer | |
feature_importances = model_trainer.compute_feature_importance() | |
st.subheader('Feature Importances') | |
fig, ax = plt.subplots() | |
# for k, v in feature_importances.items(): | |
# st.write(f"Feature importance for {k}") | |
# v.plot(kind='barh', ax=ax) | |
# ax.set_xlabel("Importance") | |
# ax.set_ylabel("Feature") | |
# ax.set_title(f"Feature Importance for {model_type}") | |
# st.pyplot(fig) | |
for k, v in feature_importances.items(): | |
# Reset index if 'v' is a Series or its index contains the feature names | |
if isinstance(v, pd.Series) or 'feature' not in v.columns: | |
v = v.reset_index() | |
v.columns = ['feature', 'score'] # Adjust column names accordingly | |
# Assuming 'v' now has columns ['feature', 'score'] | |
fig = px.bar(v, y='feature', x='score', orientation='h', | |
title=f"Feature Importance for {model_type} ({k})", | |
labels={'score': 'Importance', 'feature': 'Feature'}) | |
fig.update_layout(yaxis={'categoryorder':'total ascending'}) # Optional: This sorts the bars | |
st.plotly_chart(fig) | |
else: | |
st.error("Please generate and preprocess the dataset first.") | |
if tabs == "Economic effects": | |
st.header("Economic Effects Analysis") | |
st.write(""" | |
We can evaluate our models by looking at the Qini curves. We can use the CATE conversion model to evaluate the performance on both the Conversion and the Benefit as a function of the fraction of users targeted. | |
The Qini curve is a measure of the uplift effect of a model. It shows the difference between the uplift model and a random model. | |
""") | |
if 'uplift_sim' in st.session_state and 'model_trainer' in st.session_state: | |
df_test = st.session_state.model_trainer.df_test | |
model_type = st.radio("Choose the model type for analysis", ('Conversion Model', 'Benefit Model')) | |
# Determine which model to use based on user selection | |
if model_type == 'Conversion Model': | |
model = st.session_state.model_trainer.conversion_learner_t | |
elif model_type == 'Benefit Model': | |
model = st.session_state.model_trainer.benefit_learner_t | |
else: | |
st.error("Invalid model type selected.") | |
st.stop() | |
if model == None: | |
st.error("Please train the model first.") | |
st.stop() | |
evaluator = ModelEvaluator(model, | |
df_test, | |
X_names # df_test.columns.drop(['conversion', 'benefit', 'treatment_group_key']) | |
) | |
discounts = ['discount_05', 'discount_10', 'discount_15'] | |
qini_conversions = {} | |
qini_benefits = {} | |
for discount in discounts: | |
qini_conv, qini_ben = evaluator.eval_performance(discount) | |
qini_conversions[discount] = qini_conv | |
qini_benefits[discount] = qini_ben | |
# Plotting CATE Conversion | |
st.subheader("CATE Conversion vs Targeted Population") | |
# fig, ax_conversion = plt.subplots() | |
# for discount, color in zip(discounts, ['b', 'g', 'y']): | |
# qini_conversions[discount].plot(ax=ax_conversion, x='index', y='S', color=color) | |
# qini_conversions[discount].plot(ax=ax_conversion, x='index', y='Random', color='r', ls='--') | |
# ax_conversion.legend([f'{d} model' for d in discounts] + [f'{d} random' for d in discounts], prop={'size': 10}) | |
# ax_conversion.set_xlabel('Fraction of Targeted Users') | |
# ax_conversion.set_ylabel('CATE Conversion') | |
# ax_conversion.set_title('CATE Conversion vs Targeted Population') | |
# st.pyplot(fig) | |
# Initialize a figure object | |
fig = go.Figure() | |
# Define colors for each discount level and the random baseline | |
colors = ['blue', 'green', 'yellow'] | |
random_line_dash = 'dash' | |
# Iterate over each discount to add its line to the plot | |
for i, discount in enumerate(discounts): | |
# Add the model line | |
fig.add_trace(go.Scatter(x=qini_conversions[discount]['index'], | |
y=qini_conversions[discount]['S'], | |
mode='lines', | |
name=f'{discount} model', | |
line=dict(color=colors[i]))) | |
# Add the random baseline line | |
fig.add_trace(go.Scatter(x=qini_conversions[discount]['index'], | |
y=qini_conversions[discount]['Random'], | |
mode='lines', | |
name=f'{discount} random', | |
line=dict(color='red', dash=random_line_dash))) | |
# Update the layout of the figure | |
fig.update_layout( | |
title='CATE Conversion vs Targeted Population', | |
xaxis_title='Fraction of Targeted Users', | |
yaxis_title='CATE Conversion', | |
legend_title='Legend', | |
legend=dict( | |
x=0, | |
y=1, | |
traceorder='normal', | |
font=dict( | |
size=10, | |
) | |
) | |
) | |
# Display the figure in Streamlit | |
st.plotly_chart(fig) | |
# Plotting CATE Benefit | |
st.subheader("CATE Benefit vs Targeted Population") | |
# fig, ax_benefit = plt.subplots() | |
# for discount, color in zip(discounts, ['b', 'g', 'y']): | |
# qini_benefits[discount].plot(ax=ax_benefit, x='index', y='S', color=color) | |
# qini_benefits[discount].plot(ax=ax_benefit, x='index', y='Random', color='r', ls='--') | |
# ax_benefit.legend([f'{d} model' for d in discounts] + [f'{d} random' for d in discounts], prop={'size': 10}) | |
# ax_benefit.set_xlabel('Fraction of Targeted Users') | |
# ax_benefit.set_ylabel('CATE Benefit') | |
# ax_benefit.set_title('CATE Benefit vs Targeted Population') | |
# st.pyplot(fig) | |
# Initialize a figure object | |
fig = go.Figure() | |
# Define colors for each discount level and the random baseline | |
colors = ['blue', 'green', 'yellow'] | |
random_line_dash = 'dash' | |
# Iterate over each discount to add its line to the plot | |
for i, discount in enumerate(discounts): | |
# Add the model line | |
fig.add_trace(go.Scatter(x=qini_benefits[discount]['index'], | |
y=qini_benefits[discount]['S'], | |
mode='lines', | |
name=f'{discount} model', | |
line=dict(color=colors[i]))) | |
# Add the random baseline line | |
fig.add_trace(go.Scatter(x=qini_benefits[discount]['index'], | |
y=qini_benefits[discount]['Random'], | |
mode='lines', | |
name=f'{discount} random', | |
line=dict(color='red', dash=random_line_dash))) | |
# Update the layout of the figure | |
fig.update_layout( | |
title='CATE Benefit vs Targeted Population', | |
xaxis_title='Fraction of Targeted Users', | |
yaxis_title='CATE Benefit', | |
legend_title='Legend', | |
legend=dict( | |
x=0, | |
y=1, | |
traceorder='normal', | |
font=dict( | |
size=10, | |
) | |
) | |
) | |
# Display the figure in Streamlit | |
st.plotly_chart(fig) | |
# Plotting CATE Benefit vs CATE Conversion | |
st.subheader("CATE Benefit vs CATE Conversion") | |
# fig, ax_comp = plt.subplots() | |
# colors = ['b', 'g', 'y'] | |
# for i, discount in enumerate(discounts): | |
# qini_conc_test = pd.concat([qini_conversions[discount][['S']], qini_benefits[discount][['S']]], axis=1) | |
# qini_conc_test.columns = ['cate_conversion', 'cate_benefit'] | |
# qini_conc_test.plot(ax=ax_comp, x='cate_conversion', y='cate_benefit', color=colors[i], label=f'{discount} model') | |
# st.write('To simplify the comparison, we can plot the CATE Benefit as a function of the CATE conversion.') | |
# st.write('In the last plot for example we can see that there is a region where offering 15% discount to a targeted group of users is more efficient than giving 10% to everyone. We can obtain the same impact in overall conversion uplift while reducing our benefit loss considerably.') | |
# ax_comp.legend(prop={'size': 10}) | |
# ax_comp.set_xlabel('CATE Conversion') | |
# ax_comp.set_ylabel('CATE Benefit') | |
# ax_comp.set_title('CATE Benefit vs CATE Conversion') | |
# st.pyplot(fig) | |
# Initialize a figure object | |
fig = go.Figure() | |
# Define colors for each discount level | |
colors = ['blue', 'green', 'yellow'] | |
# Iterate over each discount to add its scatter plot to the figure | |
for i, discount in enumerate(discounts): | |
qini_conc_test = pd.concat([qini_conversions[discount]['S'], qini_benefits[discount]['S']], axis=1) | |
qini_conc_test.columns = ['cate_conversion', 'cate_benefit'] | |
# Add the scatter plot for each discount level | |
# Adjust marker size with `size` and line width with `line=dict(width=2)` | |
fig.add_trace(go.Scatter(x=qini_conc_test['cate_conversion'], | |
y=qini_conc_test['cate_benefit'], | |
mode='markers+lines', | |
name=f'{discount} model', | |
marker=dict(color=colors[i], size=6), # Adjust marker size here | |
line=dict(width=2))) # Adjust line width here | |
# Update the layout of the figure to adjust aspect ratio and margins if needed | |
fig.update_layout( | |
title='CATE Benefit vs CATE Conversion', | |
xaxis_title='CATE Conversion', | |
yaxis_title='CATE Benefit', | |
legend_title='Legend', | |
legend=dict( | |
x=0, | |
y=1, | |
traceorder='normal', | |
font=dict( | |
size=10, | |
) | |
), | |
# Optionally adjust plot and margin size for a "thinner" appearance | |
margin=dict(l=20, r=20, t=50, b=20), # Adjust margins to change plot boundary | |
height=400, # Adjust height for overall "thinness" | |
width=600 # Adjust width as needed | |
) | |
# Display the figure in Streamlit | |
st.plotly_chart(fig) | |
st.write('To simplify the comparison, we can plot the CATE Benefit as a function of the CATE conversion.') | |
st.write('In the last plot for example, we can see that there is a region where offering a 15% discount to a targeted group of users is more efficient than giving 10% to everyone. We can obtain the same impact on overall conversion uplift while reducing our benefit loss considerably.') | |
else: | |
st.error("Please ensure the model is trained and the dataset is prepared.") |