Spaces:
Sleeping
Sleeping
import streamlit as st | |
import pandas as pd | |
import plotly.express as px | |
import plotly.graph_objects as go | |
from Eda_functions import * | |
import numpy as np | |
import pickle | |
from streamlit_pandas_profiling import st_profile_report | |
import streamlit as st | |
import streamlit.components.v1 as components | |
import sweetviz as sv | |
from utilities import set_header,load_local_css | |
from st_aggrid import GridOptionsBuilder,GridUpdateMode | |
from st_aggrid import GridOptionsBuilder | |
from st_aggrid import AgGrid | |
import base64 | |
import os | |
import tempfile | |
#from ydata_profiling import ProfileReport | |
import re | |
st.set_page_config( | |
page_title="Data Validation", | |
page_icon=":shark:", | |
layout="wide", | |
initial_sidebar_state='collapsed' | |
) | |
load_local_css('styles.css') | |
set_header() | |
with open('data_import.pkl', 'rb') as f: | |
data = pickle.load(f) | |
st.session_state['cleaned_data']= data['final_df'] | |
st.session_state['category_dict'] = data['bin_dict'] | |
st.title('Data Validation and Insights') | |
target_variables=[st.session_state['category_dict'][key] for key in st.session_state['category_dict'].keys() if key =='Response Metrics'] | |
target_column = st.selectbox('Select the Target Feature/Dependent Variable (will be used in all charts as reference)',list(*target_variables)) | |
st.session_state['target_column']=target_column | |
panels=st.session_state['category_dict']['Panel Level 1'][0] | |
selected_panels=st.multiselect('Please choose the panels you wish to analyze.If no panels are selected, insights will be derived from the overall data.',st.session_state['cleaned_data'][panels].unique()) | |
aggregation_dict = {item: 'sum' if key == 'Media' else 'mean' for key, value in st.session_state['category_dict'].items() for item in value if item not in ['date','Panel_1']} | |
#st.write(st.session_state['cleaned_data']) | |
with st.expander('**Reponse Metric Analysis**'): | |
if len(selected_panels)>0: | |
st.session_state['Cleaned_data_panel']=st.session_state['cleaned_data'][st.session_state['cleaned_data']['Panel_1'].isin(selected_panels)] | |
st.session_state['Cleaned_data_panel']=st.session_state['Cleaned_data_panel'].groupby(by='date').agg(aggregation_dict) | |
st.session_state['Cleaned_data_panel']=st.session_state['Cleaned_data_panel'].reset_index() | |
else: | |
st.session_state['Cleaned_data_panel']=st.session_state['cleaned_data'].groupby(by='date').agg(aggregation_dict) | |
st.session_state['Cleaned_data_panel']=st.session_state['Cleaned_data_panel'].reset_index() | |
fig=line_plot_target(st.session_state['Cleaned_data_panel'], target=target_column, title=f'{target_column} Over Time') | |
st.plotly_chart(fig, use_container_width=True) | |
media_channel=list(*[st.session_state['category_dict'][key] for key in st.session_state['category_dict'].keys() if key =='Media']) | |
# st.write(media_channel) | |
Non_media_variables=list(*[st.session_state['category_dict'][key] for key in st.session_state['category_dict'].keys() if key =='Exogenous' or key=='Internal']) | |
st.markdown('### Annual Data Summary') | |
st.dataframe(summary(st.session_state['Cleaned_data_panel'], media_channel+[target_column], spends=None,Target=True), use_container_width=True) | |
if st.checkbox('Show raw data'): | |
st.write(pd.concat([pd.to_datetime(st.session_state['Cleaned_data_panel']['date']).dt.strftime('%m/%d/%Y'),st.session_state['Cleaned_data_panel'].select_dtypes(np.number).applymap(format_numbers)],axis=1)) | |
col1 = st.columns(1) | |
if "selected_feature" not in st.session_state: | |
st.session_state['selected_feature']=None | |
def generate_report_with_target(channel_data, target_feature): | |
report = sv.analyze([channel_data, "Dataset"], target_feat=target_feature) | |
temp_dir = tempfile.mkdtemp() | |
report_path = os.path.join(temp_dir, "report.html") | |
report.show_html(filepath=report_path, open_browser=False) # Generate the report as an HTML file | |
return report_path | |
def generate_profile_report(df): | |
pr = df.profile_report() | |
temp_dir = tempfile.mkdtemp() | |
report_path = os.path.join(temp_dir, "report.html") | |
pr.to_file(report_path) | |
return report_path | |
#st.header() | |
with st.expander('Univariate and Bivariate Report'): | |
eda_columns=st.columns(2) | |
with eda_columns[0]: | |
if st.button('Generate Profile Report',help='Univariate report which inlcudes all statistical analysis'): | |
with st.spinner('Generating Report'): | |
report_file = generate_profile_report(st.session_state['Cleaned_data_panel']) | |
if os.path.exists(report_file): | |
with open(report_file, 'rb') as f: | |
st.success('Report Generated') | |
st.download_button( | |
label="Download EDA Report", | |
data=f.read(), | |
file_name="pandas_profiling_report.html", | |
mime="text/html" | |
) | |
else: | |
st.warning("Report generation failed. Unable to find the report file.") | |
with eda_columns[1]: | |
if st.button('Generate Sweetviz Report',help='Bivariate report for selected response metric'): | |
with st.spinner('Generating Report'): | |
report_file = generate_report_with_target(st.session_state['Cleaned_data_panel'], target_column) | |
if os.path.exists(report_file): | |
with open(report_file, 'rb') as f: | |
st.success('Report Generated') | |
st.download_button( | |
label="Download EDA Report", | |
data=f.read(), | |
file_name="report.html", | |
mime="text/html" | |
) | |
else: | |
st.warning("Report generation failed. Unable to find the report file.") | |
#st.warning('Work in Progress') | |
with st.expander('Media Variables Analysis'): | |
# Get the selected feature | |
st.session_state["selected_feature"]= st.selectbox('Select media', [col for col in media_channel if 'cost' not in col.lower() and 'spend' not in col.lower()]) | |
# Filter spends features based on the selected feature | |
spends_features = [col for col in st.session_state['Cleaned_data_panel'].columns if any(keyword in col.lower() for keyword in ['cost', 'spend'])] | |
spends_feature = [col for col in spends_features if re.split(r'_cost|_spend', col.lower())[0] in st.session_state["selected_feature"]] | |
if 'validation' not in st.session_state: | |
st.session_state['validation']=[] | |
val_variables=[col for col in media_channel if col!='date'] | |
if len(spends_feature)==0: | |
st.warning('No spends varaible available for the selected metric in data') | |
else: | |
fig_row1 = line_plot(st.session_state['Cleaned_data_panel'], x_col='date', y1_cols=[st.session_state["selected_feature"]], y2_cols=[target_column], title=f'Analysis of {st.session_state["selected_feature"]} and {[target_column][0]} Over Time') | |
st.plotly_chart(fig_row1, use_container_width=True) | |
st.markdown('### Summary') | |
st.dataframe(summary(st.session_state['cleaned_data'],[st.session_state["selected_feature"]],spends=spends_feature[0]),use_container_width=True) | |
cols2=st.columns(2) | |
with cols2[0]: | |
if st.button('Validate'): | |
st.session_state['validation'].append(st.session_state["selected_feature"]) | |
with cols2[1]: | |
if st.checkbox('Validate all'): | |
st.session_state['validation'].extend(val_variables) | |
st.success('All media variables are validated ✅') | |
if len(set(st.session_state['validation']).intersection(val_variables))!=len(val_variables): | |
validation_data=pd.DataFrame({'Validate':[True if col in st.session_state['validation'] else False for col in val_variables], | |
'Variables':val_variables | |
}) | |
cols3=st.columns([1,30]) | |
with cols3[1]: | |
validation_df=st.data_editor(validation_data, | |
# column_config={ | |
# 'Validate':st.column_config.CheckboxColumn(wi) | |
# }, | |
column_config={ | |
"Validate": st.column_config.CheckboxColumn( | |
default=False, | |
width=100, | |
), | |
'Variables':st.column_config.TextColumn( | |
width=1000 | |
) | |
},hide_index=True) | |
selected_rows = validation_df[validation_df['Validate']==True]['Variables'] | |
#st.write(selected_rows) | |
st.session_state['validation'].extend(selected_rows) | |
not_validated_variables = [col for col in val_variables if col not in st.session_state["validation"]] | |
if not_validated_variables: | |
not_validated_message = f'The following variables are not validated:\n{" , ".join(not_validated_variables)}' | |
st.warning(not_validated_message) | |
with st.expander('Non Media Variables Analysis'): | |
selected_columns_row4 = st.selectbox('Select Channel',Non_media_variables,index=1) | |
# # Create the dual-axis line plot | |
fig_row4 = line_plot(st.session_state['Cleaned_data_panel'], x_col='date', y1_cols=[selected_columns_row4], y2_cols=[target_column], title=f'Analysis of {selected_columns_row4} and {target_column} Over Time') | |
st.plotly_chart(fig_row4, use_container_width=True) | |
selected_non_media=selected_columns_row4 | |
sum_df = st.session_state['Cleaned_data_panel'][['date', selected_non_media,target_column]] | |
sum_df['Year']=pd.to_datetime(st.session_state['Cleaned_data_panel']['date']).dt.year | |
#st.dataframe(df) | |
#st.dataframe(sum_df.head(2)) | |
sum_df=sum_df.groupby('Year').agg('sum') | |
sum_df.loc['Grand Total']=sum_df.sum() | |
sum_df=sum_df.applymap(format_numbers) | |
sum_df.fillna('-',inplace=True) | |
sum_df=sum_df.replace({"0.0":'-','nan':'-'}) | |
st.markdown('### Summary') | |
st.dataframe(sum_df,use_container_width=True) | |
with st.expander('Correlation Analysis'): | |
options = list(st.session_state['Cleaned_data_panel'].select_dtypes(np.number).columns) | |
# selected_options = [] | |
# num_columns = 4 | |
# num_rows = -(-len(options) // num_columns) # Ceiling division to calculate rows | |
# # Create a grid of checkboxes | |
# st.header('Select Features for Correlation Plot') | |
# tick=False | |
# if st.checkbox('Select all'): | |
# tick=True | |
# selected_options = [] | |
# for row in range(num_rows): | |
# cols = st.columns(num_columns) | |
# for col in cols: | |
# if options: | |
# option = options.pop(0) | |
# selected = col.checkbox(option,value=tick) | |
# if selected: | |
# selected_options.append(option) | |
# # Display selected options | |
selected_options=st.multiselect('Select Variables For correlation plot',[var for var in options if var!= target_column],default=options[3]) | |
st.pyplot(correlation_plot(st.session_state['Cleaned_data_panel'],selected_options,target_column)) | |