from ast import arg
import streamlit as st
import pandas as pd
import PIL
import re
from io import StringIO
import boto3
from urlextract import URLExtract
import time
# from joblib import dump, load
import joblib
from bokeh.models.widgets import Div
import email
import os
#from ipyfilechooser import FileChooser
#from IPython.display import display
from io import BytesIO
from bs4 import BeautifulSoup
import matplotlib.pyplot as plt
import numpy as np
import timeit
import shutil
CURRENT_THEME = "blue"
IS_DARK_THEME = True
def table_data():
# creating table data
field = [
'Data Scientist',
'Dataset',
'Algorithm',
'Framework',
'Ensemble',
'Domain',
'Model Size'
]
data = [
'Chen Song',
'Internal + Campaign monitor',
'Random Forest',
'Sci-kit learn',
'Bootstrapping',
'Bootstrapping Aggregation',
'4 KB'
]
data = {
'Field': field,
'Data': data
}
df = pd.DataFrame.from_dict(data)
return df
def url_button(button_name, url):
if st.button(button_name):
js = """window.open('{url}')""".format(url=url) # New tab or window
html = '
'.format(js)
div = Div(text=html)
st.bokeh_chart(div)
def get_industry_code_dict(training_dataset):
training_dataset['industry_code'] = training_dataset['industry'].astype(
'category')
cat_columns = training_dataset.select_dtypes(['category']).columns
training_dataset[cat_columns] = training_dataset[cat_columns].apply(
lambda x: x.cat.codes)
industry_code_dict = dict(
zip(training_dataset.industry, training_dataset.industry_code))
return industry_code_dict
def parse_email(uploaded_file):
parsed_email = []
efile = open(uploaded_file.name,'r')
emailstr = ""
for i, line in enumerate(efile):
emailstr += line
b = email.message_from_string(emailstr)
for part in b.walk():
if part.get_content_type():
body = str(part.get_payload())
soup = BeautifulSoup(body)
paragraphs = soup.find_all('body')
for paragraph in paragraphs:
parsed_email.append(paragraph.text)
return parsed_email
#def email_upload():
# print("Please upload your email (In HTML Format)")
# upload = FileUpload(accept='.html', multiple=True)
# display(upload)
# return upload
# fc = FileChooser()
# display(fc)
# return fc
# New - In-Use
def email_extractor(email_uploaded):
parse = parse_email(email_uploaded)
email_text = ''.join(parse).strip()
# extract the email body using string manipulation functions
email_body_start_index = email_text.find('Bright Apps LLC')
email_body_end_index = email_text.find('To read more')
email_body = email_text[email_body_start_index:email_body_end_index].strip()
# get rid of non-text elements
email_body = email_body.replace('\n', '')
email_body = email_body.replace('\t', '')
email_body = email_body.replace('\r', '')
email_body = email_body.replace('', '')
email_body = email_body.replace('', '')
email_body = email_body.replace('\xa0', '')
# find length of URLs if any
extractor = URLExtract()
urls = extractor.find_urls(email_body)
url_cnt = len(urls)
# remove URLs and get character count
body = re.sub(r'\w+:\/{2}[\d\w-]+(\.[\d\w-]+)*(?:(?:\/[^\s/]*))*', '', email_body)
sep = '©'
body = body.split(sep, 1)[0]
character_cnt = sum(not chr.isspace() for chr in body)
return email_body, character_cnt, url_cnt
# extract email body from parse email
def email_body_extractor(email_data):
# email_data = parsed_email.data[0]
emailstr = email_data.decode("utf-8")
b = email.message_from_string(emailstr)
body = ""
if b.is_multipart():
for part in b.walk():
ctype = part.get_content_type()
cdispo = str(part.get('Content-Disposition'))
# skip any text/plain (txt) attachments
if ctype == 'text/plain' and 'attachment' not in cdispo:
body = part.get_payload() # decode
break
# not multipart - i.e. plain text, no attachments, keeping fingers crossed
else:
body = b.get_payload()
# Remove escape sequences
body = body.replace('\n', '')
body = body.replace('\t', '')
body = body.replace('\r', '')
body = body.replace('', '')
body = body.replace('', '')
# Extract urls in the email body and get url counts
extractor = URLExtract()
urls = extractor.find_urls(body)
url_cnt = len(urls)
# Remove urls
body = re.sub(
r'\w+:\/{2}[\d\w-]+(\.[\d\w-]+)*(?:(?:\/[^\s/]*))*', '', body)
sep = '©'
body = body.split(sep, 1)[0]
character_cnt = sum(not chr.isspace() for chr in body)
return body, character_cnt, url_cnt
def add_bg_from_url():
st.markdown(
f"""
""",
unsafe_allow_html=True
)
add_bg_from_url()
#linear-gradient(0deg,#010405 0,#061c2c 55%,#0a3144 75%,#0f4d60)
st.markdown("# Character Count: Email Industry")
stats_col1, stats_col2, stats_col3, stats_col4 = st.columns([1, 1, 1, 1])
with stats_col1:
st.caption("Production: Ready")
with stats_col2:
st.caption("Accuracy: 85%")
with stats_col3:
st.caption("Speed: 16.89 ms")
with stats_col4:
st.caption("Industry: Email")
with st.sidebar:
with st.expander('Model Description', expanded=False):
img = PIL.Image.open("figures/ModelCC.png")
st.image(img)
st.markdown('Finding the correct length for an email campaign to maximize user engagement can be an ambiguous task. The Loxz Character Count Model allows you to predict the correct length of your emails for a particular industry and a particular type of email. Using these inputs and trained on an extensive proprietary data set from the Loxz family digital archive, the models incorporate real-world and synthetic data to find the optimized character counts. We applied the random forest algorithm in this model. Bootstrapping was also ensembled in the algorithm which effectively prevents overfitting by reducing variance. The model achieves an 86% accuracy on the test set. This inference-based ML model will help the campaign engineers start with an acceptable length and zero in on the best character count, maximizing engagement in their campaign.')
with st.expander('Model Information', expanded=False):
hide_table_row_index = """
"""
st.markdown(hide_table_row_index, unsafe_allow_html=True)
st.table(table_data())
url_button('Model Homepage', 'https://www.loxz.com/#/models/CTA')
# url_button('Full Report','https://resources.loxz.com/reports/realtime-ml-character-count-model')
url_button('Amazon Market Place', 'https://aws.amazon.com/marketplace')
industry_lists = [
'Retail',
'Software and Technology',
'Hospitality',
'Academic and Education',
'Healthcare',
'Energy',
'Real Estate',
'Entertainment',
'Finance and Banking'
]
campaign_types = [
'Promotional',
'Transactional',
'Webinar',
'Survey',
'Newsletter',
'Engagement',
'Usage_and_Consumption',
'Review_Request',
'Product_Announcement',
'Abandoned_Cart'
]
target_variables = [
'conversion_rate',
'click_to_open_rate'
]
uploaded_file = st.file_uploader(
"Please upload your email (In HTML Format)", type=["html"])
def save_file(uploaded_file):
with open(os.path.join("./",uploaded_file.name),"wb") as f:
f.write(uploaded_file.getbuffer())
if uploaded_file is None:
# upload_img = PIL.Image.open(uploaded_file)
upload_img = None
# else:
# upload_img = None
industry = st.selectbox(
'Please select your industry',
industry_lists,
index=6
)
campaign = st.selectbox(
'Please select your campaign type',
campaign_types,
index=5
)
target = st.selectbox(
'Please select your target variable',
target_variables,
index=1
)
st.markdown("""---""")
#char_reco_preference = st.selectbox(
# 'Do you want to increase or decrease your character count in the email?',
# ["Increase", "Decrease"],
# index=1)
def get_files_from_aws(bucket, prefix):
"""
get files from aws s3 bucket
bucket (STRING): bucket name
prefix (STRING): file location in s3 bucket
"""
s3_client = boto3.client('s3',
aws_access_key_id=st.secrets["aws_id"],
aws_secret_access_key=st.secrets["aws_key"])
file_obj = s3_client.get_object(Bucket=bucket, Key=prefix)
body = file_obj['Body']
string = body.read().decode('utf-8')
df = pd.read_csv(StringIO(string))
return df
# st.info([industry,campaign,target,char_reco_preference])
if st.button('Generate Predictions'):
start_time = time.time()
if uploaded_file is None:
st.error('Please upload a email (HTML format)')
else:
save_file(uploaded_file)
placeholder = st.empty()
placeholder.text('Loading Data')
# Starting predictions
model = joblib.load('models/models.sav')
# Generate Email Data
email_data = get_files_from_aws(
'emailcampaigntrainingdata', 'trainingdata/email_dataset_training.csv')
acc_data = get_files_from_aws(
'emailcampaigntrainingdata', 'trainingdata/email_dataset_training_raw.csv')
email_data_ = email_data[["email_body", "industry", "campaign_type",
"character_cnt", "url_cnt", "Open_Rate", "Click_Through_Rate"]]
email_data_ = email_data_.rename(
{'Open_Rate': 'Click-to-open_Rate', 'Click_Through_Rate': 'Conversion_Rate'})
df_email_data = email_data_.rename(
columns={'Open_Rate': 'Click-to-open_Rate', 'Click_Through_Rate': 'Conversion_Rate'})
# Dataset:
training_dataset = get_files_from_aws(
'emailcampaigntrainingdata', 'modelCC/training.csv')
# X_test = get_files_from_aws('emailcampaigntrainingdata','modelCC/Xtest.csv')
# Y_test = get_files_from_aws('emailcampaigntrainingdata','modelCC/ytest.csv')
# print("Getting Data Time: %s seconds" % (time.time() - start_time))
industry_code_dict = get_industry_code_dict(email_data)
#uploaded_file = FileChooser(uploaded_file)
#bytes_data = uploaded_file.getvalue()
email_body, character_cnt, url_cnt = email_extractor(uploaded_file)
# Start the prediction
# Need to solve X test issue
# y_pred = model.predict(X_test)
df_uploaded = pd.DataFrame(
columns=['character_cnt', "url_cnt", "industry"])
df_uploaded.loc[0] = [character_cnt, url_cnt, industry]
df_uploaded["industry_code"] = industry_code_dict.get(industry)
df_uploaded_test = df_uploaded[[
"industry_code", "character_cnt", "url_cnt"]]
predicted_rate = model.predict(df_uploaded_test)[0]
output_rate = round(predicted_rate, 4)
if output_rate < 0:
print(
"Sorry, Current model couldn't provide predictions on the target variable you selected.")
else:
st.markdown('##### Current Character Count in Your Email is: {}'.format(
character_cnt), unsafe_allow_html=True)
# st.info('The model predicts that it achieves a {} of {}%'.format(target, str(round(output_rate*100,2))))
if target == 'conversion_rate':
target_vis = 'Click_Through_Rate'
else:
target_vis = 'Open_Rate'
st.markdown('##### The model predicts that it achieves a {} of {}%'.format(
target_vis, str(round(output_rate*100, 3))), unsafe_allow_html=True)
selected_industry_code = industry_code_dict.get(industry)
if target == "click_to_open_rate":
selected_variable = "Open_Rate"
if target == "conversion_rate":
selected_variable = "Click_Through_Rate"
df_reco = training_dataset[[
"industry_code", "character_cnt", "url_cnt", selected_variable]]
df_reco = df_reco[df_reco["industry_code"]
== selected_industry_code]
df_reco[selected_variable] = df_reco[selected_variable].apply(
lambda x: round(x, 3))
df_reco_sort = df_reco.sort_values(by=[selected_variable])
df_reco = df_reco.drop_duplicates(subset=selected_variable)
#preference = char_reco_preference
#if preference == "Increase":
# df_reco_opt = df_reco[(df_reco[selected_variable] > output_rate) & (
# df_reco["character_cnt"] > character_cnt) & (df_reco["character_cnt"] <= (1.5*character_cnt))]
# df_reco_opt_rank = df_reco_opt.nlargest(3, [selected_variable])
# decrease character reco
#if preference == "Decrease":
# df_reco_opt = df_reco[(df_reco[selected_variable] > output_rate) & (
# df_reco["character_cnt"] < character_cnt)]
# df_reco_opt_rank = df_reco_opt.nlargest(3, [selected_variable])
# split into two dataframes of higher and lower character_cnt (added apr 2023)
char_cnt_uploaded = character_cnt
df_reco_opt1 = df_reco[(df_reco[selected_variable] > output_rate) & (df_reco["character_cnt"] > char_cnt_uploaded) & (df_reco["character_cnt"] <= (1.5*char_cnt_uploaded))]
df_reco_opt2 = df_reco[(df_reco[selected_variable] > output_rate) & (df_reco["character_cnt"] < char_cnt_uploaded) & (df_reco["character_cnt"] >= (char_cnt_uploaded/2))]
# drop duplicates of character_cnt keeping the row with the highest output_rate
df_reco_opt1 = df_reco_opt1.sort_values(by=[selected_variable], ascending=False).drop_duplicates(subset=["character_cnt"])
df_reco_opt2 = df_reco_opt2.sort_values(by=[selected_variable], ascending=False).drop_duplicates(subset=["character_cnt"])
# get top 2 largest in higher and lower dataframe
df_reco_opt_rank1 = df_reco_opt1.nlargest(2, [selected_variable])
df_reco_opt_rank2 = df_reco_opt2.nlargest(2, [selected_variable])
df_reco_opt_rank = pd.concat([df_reco_opt_rank1, df_reco_opt_rank2])
df_reco_opt_rank = df_reco_opt_rank.nlargest(3,[selected_variable])
if selected_variable == "Open_Rate":
selected_variable = "Click-to-Open_Rate"
if selected_variable == "Click_Through_Rate":
selected_variable = "Conversion_Rate"
st.markdown('##### To get higher, {}, the model recommends the following options:'.format(
selected_variable), unsafe_allow_html=True)
if len(df_reco_opt_rank) == 0:
st.markdown('##### You ve already achieved the highest, {}, with the current character count!'.format(
selected_variable), unsafe_allow_html=True)
else:
#for _, row in df_reco_opt_rank.iterrows():
# Character_Count = row[1]
# selected_variable = row[3]
# print(f"·Number of Characters: {int(Character_Count)}, Target Rate: {round(selected_variable, 3)*100}", "%")
# st.markdown('Number of Characters: {}, Target Rate: {}'.format(
# int(Character_Count), round(selected_variable*100, 3)))
chars = []
sel_var_values = []
for _, row in df_reco_opt_rank.iterrows():
Character_Count = row[1]
selected_variable_number = row[3]
chars.append(int(Character_Count))
sel_var_values.append(round(selected_variable_number, 3)*100)
st.write(f"·Number of Characters: {int(Character_Count)}, Target Rate: {round(round(selected_variable_number, 3)*100, 3)}", "%")
st.write("\n")
if len(chars) > 1:
#fig = plt.figure()
#ax = fig.add_axes([0,0,1,1])
fig, ax = plt.subplots(figsize=(10,4))
bars = ax.barh(np.arange(len(chars)), sel_var_values, height=0.175, color='#0F4D60')
#ax.bar_label(bars)
ax.set_yticks(np.arange(len(chars)))
ax.set_yticklabels(np.array(chars), fontsize=14)
ax.set_title('Character Counts vs. Target Variable Rates', fontsize=18)
ax.set_ylabel('Character Counts', fontsize=16)
ax.set_xlabel('Target Rates %', fontsize=16)
for i, bar in enumerate(bars):
rounded_value = round(sel_var_values[i], 2)
ax.text(bar.get_width() + 0.3, bar.get_y() + bar.get_height()/2, str(rounded_value) + '%', ha='left', va='center', fontsize=12, fontweight='bold')
ax.margins(0.1,0.05)
biggest_bar_index = np.argmax(sel_var_values)
bars[biggest_bar_index].set_color('#00BF93')
st.plotly_chart(fig, use_container_width=True)
st.write("\n")
#st.write(np.array(chars))
chars_out = dict(zip(chars, sel_var_values))
sorted_chars_out = sorted(chars_out.items(), key=lambda x: x[1], reverse=True)
placeholder.empty()
#st.write(time.time() - start_time)