working model CC

app.py

@@ -1,9 +1,55 @@
+from ast import arg
 import streamlit as st
 import pandas as pd
 import PIL
+import re
+from io import StringIO
+import boto3
+
+# from joblib import dump, load
+
+import joblib
 
 from bokeh.models.widgets import Div
 
+import email
+from urlextract import URLExtract
+
+import main_app
+
+
+def table_data():
+    # creating table data
+    field = [
+        'Data Scientist',
+        'Dataset',
+        'Algorithm',
+        'Framework',
+        'Ensemble',
+        'Domain',
+        'Model Size'
+    ]
+
+    data = [
+        'Chen Song',
+        'Internal + Campaign monitor',
+        'Random Forest',
+        'Sci-kit learn',
+        'Bootstrapping',
+        'Bootstrapping Aggregation',
+        '4 KB'
+    ]
+
+    data = {
+        'Field':field,
+        'Data':data
+    }
+
+    df = pd.DataFrame.from_dict(data)
+
+    return df
+
+
 
 def url_button(button_name,url):
     if st.button(button_name):
@@ -12,20 +58,89 @@ def url_button(button_name,url):
         div = Div(text=html)
         st.bokeh_chart(div)
 
+def get_industry_code_dict(training_dataset):
+    training_dataset['industry_code'] = training_dataset['industry'].astype('category')
+    cat_columns = training_dataset.select_dtypes(['category']).columns
+    training_dataset[cat_columns] = training_dataset[cat_columns].apply(lambda x: x.cat.codes)
+    industry_code_dict = dict(zip(training_dataset.industry, training_dataset.industry_code))
+    return industry_code_dict
+
 
+## extract email body from parse email
+def email_body_extractor(email_data):   
+    # email_data = parsed_email.data[0]
+    emailstr = email_data.decode("utf-8") 
+    b = email.message_from_string(emailstr)
+    body = ""
 
-st.markdown("# Character Counter: Email Industry")
+    if b.is_multipart():
+        for part in b.walk():
+            ctype = part.get_content_type()
+            cdispo = str(part.get('Content-Disposition'))
 
-col1, col2, col3 = st.columns([1,1,1])
+            # skip any text/plain (txt) attachments
+            if ctype == 'text/plain' and 'attachment' not in cdispo:
+                body = part.get_payload()  # decode
+                break
+    # not multipart - i.e. plain text, no attachments, keeping fingers crossed
+    else:
+        body = b.get_payload()
+    ## Remove escape sequences
+    body = body.replace('\n', '')
+    body = body.replace('\t', '')
+    body = body.replace('\r', '')
+    body = body.replace('</b>', '')
+    body = body.replace('<b>', '')
 
-with col2:
-    img = PIL.Image.open("figures/ModelCC.png")
-    st.image(img)
+    
+    ## Extract urls in the email body and get url counts
+    extractor = URLExtract()
+    urls = extractor.find_urls(body)
+    url_cnt = len(urls)
+    ## Remove urls
+    body = re.sub(r'\w+:\/{2}[\d\w-]+(\.[\d\w-]+)*(?:(?:\/[^\s/]*))*', '', body)    
+    sep = '©'
+    body = body.split(sep, 1)[0]
+    character_cnt = sum(not chr.isspace() for chr in body)
+
+    return body, character_cnt, url_cnt
+
+# def select_char_preference_variables():
+#     opt_list = ["Increase", "Decrease"]
+#     button_option = widgets.RadioButtons(options = opt_list)
+#     print("Do you want to increase or decrease your character count in the email?")
+#     display(button_option)
+#     return button_option
+
+
+def add_bg_from_url():
+    st.markdown(
+         f"""
+         <style>
+         .stApp {{
+             background-image: linear-gradient(#0A3144,#126072,#1C8D99);
+             background-attachment: fixed;
+             background-size: cover
+         }}
+         </style>
+         """,
+         unsafe_allow_html=True
+     )
+
+# add_bg_from_url() 
+
+st.markdown("# Character Count: Email Industry")
+
+# col1, col2, col3 = st.columns([1,1,1])
+
+# with col2:
+#     img = PIL.Image.open("figures/ModelCC_solid.png")
+#     st.image(img)
 
 stats_col1, stats_col2, stats_col3, stats_col4 = st.columns([1,1,1,1])
 
 with stats_col1:
-    st.metric(label="Production", value="Ready")
+    st.metric(label="Production", value="Production")
 with stats_col2:
     st.metric(label="Accuracy", value="85%")
 
@@ -36,15 +151,26 @@ with stats_col4:
     st.metric(label="Industry", value="Email")
 
 
-with st.sidebar:
 
+with st.sidebar:
 
     with st.expander('Model Description', expanded=False):
+        img = PIL.Image.open("figures/ModelCC.png")
+        st.image(img)
         st.markdown('Finding the correct length for an email campaign to maximize user engagement can be an ambiguous task. The Loxz Character Count Model allows you to predict the correct length of your emails for a particular industry and a particular type of email. Using these inputs and trained on an extensive proprietary data set from the Loxz family digital archive, the models incorporate real-world and synthetic data to find the optimized character counts. We applied the random forest algorithm in this model. Bootstrapping was also ensembled in the algorithm which effectively prevents overfitting by reducing variance. The model achieves an 86% accuracy on the test set. This inference-based ML model will help the campaign engineers start with an acceptable length and zero in on the best character count, maximizing engagement in their campaign.')
 
+    with st.expander('Model Information', expanded=False):
+        hide_table_row_index = """
+            <style>
+            thead tr th:first-child {display:none}
+            tbody th {display:none}
+            </style>
+            """
+        st.markdown(hide_table_row_index, unsafe_allow_html=True)
+        st.table(table_data())
 
-    url_button('Model Homepage','https://www.loxz.com/#/models/CC')
-    url_button('Full Report','https://resources.loxz.com/reports/realtime-ml-character-count-model')
+    url_button('Model Homepage','https://www.loxz.com/#/models/CTA')
+    # url_button('Full Report','https://resources.loxz.com/reports/realtime-ml-character-count-model')
     url_button('Amazon Market Place','https://aws.amazon.com/marketplace')
 
 
@@ -52,12 +178,12 @@ industry_lists = [
     'Retail',
     'Software and Technology',
     'Hospitality',
-    'Adacemic and Education',
+    'Academic and Education',
     'Healthcare',
     'Energy',
     'Real Estate',
     'Entertainment',
-    'Fianance and Banking'
+    'Finance and Banking'
 ]
 
 campaign_types = [
@@ -74,12 +200,19 @@ campaign_types = [
 ]
 
 target_variables = [
-    'open_rate', 
-    'click_through_rate', 
-    'abandoned_cart', 
-    'unsubscribe_rate'
+    'conversion_rate', 
+    'click_to_open_rate'
 ]
 
+uploaded_file = st.file_uploader("Please upload your email (In HTML Format)", type=["html"])
+
+if uploaded_file is None:
+    # upload_img = PIL.Image.open(uploaded_file)
+    upload_img = None
+# else:
+    # upload_img = None
+
+
 industry = st.selectbox(
     'Please select your industry',
     industry_lists
@@ -94,3 +227,117 @@ target = st.selectbox(
     'Please select your target variable',
     target_variables
 )
+
+st.markdown("""---""")
+
+char_reco_preference = st.selectbox(
+    'Do you want to increase or decrease your character count in the email?',
+    ["Increase", "Decrease"])
+
+
+def get_files_from_aws(bucket,prefix):
+    """
+        get files from aws s3 bucket
+    
+    bucket (STRING): bucket name
+    prefix (STRING): file location in s3 bucket
+    """
+    s3_client = boto3.client('s3',
+        aws_access_key_id = st.secrets["aws_id"],
+        aws_secret_access_key = st.secrets["aws_key"])
+
+    file_obj = s3_client.get_object(Bucket=bucket,Key=prefix)
+    body = file_obj['Body']
+    string = body.read().decode('utf-8')
+    
+    df = pd.read_csv(StringIO(string))
+    
+    return df 
+
+
+
+
+if st.button('Generate Predictions'):
+    if uploaded_file is None:
+        st.error('Please upload a email (HTML format)')
+    else:
+        placeholder = st.empty()
+        placeholder.text('Loading Data')
+
+        # Starting predictions
+        model = joblib.load('models/models.sav')
+        print(type(model))
+        # Generate Email Data
+        email_data = get_files_from_aws('emailcampaigntrainingdata','trainingdata/email_dataset_training.csv')
+        acc_data = get_files_from_aws('emailcampaigntrainingdata','trainingdata/email_dataset_training_raw.csv')
+
+        email_data_ = email_data[["email_body", "industry", "campaign_type","character_cnt", "url_cnt","Open_Rate", "Click_Through_Rate"]]
+        email_data_ = email_data_.rename({'Open_Rate': 'Click-to-open_Rate', 'Click_Through_Rate': 'Conversion_Rate'})
+        df_email_data = email_data_.rename(columns={'Open_Rate': 'Click-to-open_Rate', 'Click_Through_Rate': 'Conversion_Rate'})
+
+        # Dataset:
+        training_dataset = get_files_from_aws('emailcampaigntrainingdata','modelCC/training.csv')
+        X_test = get_files_from_aws('emailcampaigntrainingdata','modelCC/Xtest.csv')
+        Y_test = get_files_from_aws('emailcampaigntrainingdata','modelCC/ytest.csv')
+
+        
+        industry_code_dict = get_industry_code_dict(email_data)
+
+        bytes_data = uploaded_file.getvalue()
+
+        email_body, character_cnt, url_cnt = email_body_extractor(bytes_data)
+
+        # Start the prediction
+        # Need to solve X test issue
+
+        # y_pred = model.predict(X_test)
+        df_uploaded = pd.DataFrame(columns=['character_cnt', "url_cnt", "industry"])
+        df_uploaded.loc[0] = [character_cnt, url_cnt, industry]
+        df_uploaded["industry_code"] = industry_code_dict.get(industry)
+        df_uploaded_test = df_uploaded[["industry_code", "character_cnt", "url_cnt"]]
+        predicted_rate =  model.predict(df_uploaded_test)[0]
+        output_rate = round(predicted_rate*100,2)
+        print(output_rate)
+        # output_rate = 0.5
+        if output_rate < 0:
+            print("Sorry, Current model couldn't provide predictions on the target variable you selected.")
+        else:
+            st.info('Current Character Count in Your Email is: {}'.format(character_cnt))
+            st.info('The model predicts that it achieves a {} of {}%'.format(target, str(output_rate)))
+            
+            # print(target)
+            if target == "click_to_open_rate":
+                selected_variable = "Open_Rate"
+            if target == "conversion_rate":
+                selected_variable = "Click_Through_Rate"
+
+            df_reco = training_dataset[["industry_code", "character_cnt", "url_cnt", selected_variable]]  
+            df_reco = df_reco[df_reco["industry_code"] == industry]
+            df_reco[selected_variable]=df_reco[selected_variable].apply(lambda x:round(x, 3))
+            df_reco_sort = df_reco.sort_values(by=[selected_variable])
+            df_reco = df_reco.drop_duplicates(subset=selected_variable)
+
+            if char_reco_preference == "Increase":
+                df_reco_opt = df_reco[(df_reco[selected_variable] > output_rate) & (df_reco["character_cnt"] > character_cnt) & (df_reco["character_cnt"] <= (1.5*character_cnt))]
+                df_reco_opt_rank = df_reco_opt.nlargest(3,[selected_variable])
+            else:
+                df_reco_opt = df_reco[(df_reco[selected_variable] > output_rate) & (df_reco["character_cnt"] < character_cnt)]
+                df_reco_opt_rank = df_reco_opt.nlargest(3,[selected_variable])
+                
+            if selected_variable == "Open_Rate":
+                selected_variable = "Click-to-Open_Rate"
+            if selected_variable == "Click_Through_Rate":
+                selected_variable = "Conversion_Rate"
+
+            st.info('To get higher, {},the model recommends the following options:'.format(selected_variable))
+            if len(df_reco_opt_rank) == 0:
+                st.info('You ve already achieved the highest, {}, with the current character count!'.format(selected_variable))
+            else:
+                for _, row in df_reco_opt_rank.iterrows():
+                    Character_Count = row[1]
+                    selected_variable = row[3]
+                    # print(f"·Number of Characters: {int(Character_Count)}, Target Rate: {round(selected_variable, 3)*100}", "%")
+                    st.info('Number of Characters: {}, Target Rate: {}'.format(int(Character_Count), round(selected_variable, 3)*100))
+
+
+        placeholder.empty()

main_app.py

@@ -0,0 +1,15 @@
+import streamlit as st
+
+st.set_page_config(layout="wide")
+
+st.markdown(
+    """
+<style>
+body {
+    background-image: linear-gradient(#2e7bcf,#2e7bcf);
+    color: white;
+}
+</style>
+""",
+    unsafe_allow_html=True,
+)

requirements.txt

@@ -1 +1,2 @@
 bokeh==2.4.1
+joblib