Upload 4 files

dump/1_Transformations.py

@@ -0,0 +1,449 @@
+import streamlit as st
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+from Eda_functions import format_numbers
+import numpy as np
+import pickle
+from st_aggrid import AgGrid
+from st_aggrid import GridOptionsBuilder,GridUpdateMode
+from utilities import set_header,load_local_css
+from st_aggrid import GridOptionsBuilder
+import time
+import itertools
+import statsmodels.api as sm
+import numpy as npc
+import re
+import itertools
+from sklearn.metrics import mean_absolute_error, r2_score,mean_absolute_percentage_error  
+from sklearn.preprocessing import MinMaxScaler
+import os
+import matplotlib.pyplot as plt
+from statsmodels.stats.outliers_influence import variance_inflation_factor
+st.set_option('deprecation.showPyplotGlobalUse', False)
+from datetime import datetime
+import seaborn as sns
+from Data_prep_functions import *
+
+st.set_page_config(
+  page_title="Model Build",
+  page_icon=":shark:",
+  layout="wide",
+  initial_sidebar_state='collapsed'
+)
+
+load_local_css('styles.css')
+set_header()
+
+
+st.title('1. Build Your Model')
+
+# media_data=pd.read_csv('Media_data_for_model.csv')
+media_data=pd.read_csv('Media_data_for_model_dma_level.csv')
+date=media_data['Date']
+st.session_state['date']=date
+revenue=media_data['Total Approved Accounts - Revenue']
+media_data.drop(['Total Approved Accounts - Revenue'],axis=1,inplace=True)
+media_data.drop(['Date'],axis=1,inplace=True)
+media_data.reset_index(drop=True,inplace=True)
+media_data.dropna(inplace=True)
+
+if st.toggle('Apply Transformations on DMA/Panel Level'):
+  dma=st.selectbox('Select the Level of data ',[ col for col in media_data.columns if col.lower() in ['dma','panel']])
+
+
+else:
+  """ code to aggregate data on date """
+
+
+  dma=None
+
+# dma_dict={ dm:media_data[media_data[dma]==dm] for dm in media_data[dma].unique()}
+# st.write(dma_dict)
+
+st.markdown('## Select the Range of Transformations')
+columns = st.columns(2)
+old_shape=media_data.shape
+
+
+if "old_shape" not in st.session_state:
+   st.session_state['old_shape']=old_shape
+
+
+with columns[0]:
+  slider_value_adstock  = st.slider('Select Adstock Range (only applied to media)', 0.0, 1.0, (0.2, 0.4), step=0.1, format="%.2f")
+with columns[1]:
+  slider_value_lag = st.slider('Select Lag Range (applied to media, seasonal, macroeconomic variables)', 1, 7, (1, 3), step=1)
+
+# with columns[2]:
+#    slider_value_power=st.slider('Select Power range (only applied to media )',0,4,(1,2),step=1)
+
+# with columns[1]:
+#    st.number_input('Select the range of half saturation point ',min_value=1,max_value=5)
+#    st.number_input('Select the range of  ')
+
+def lag(data,features,lags,dma=None):
+    if dma:
+        
+        transformed_data=pd.concat([data.groupby([dma])[features].shift(lag).add_suffix(f'_lag_{lag}') for lag in lags],axis=1)
+        transformed_data=transformed_data.fillna(method='bfill')
+        return pd.concat([transformed_data,data],axis=1)
+
+    else:
+                          
+        ''' data should be aggregated on date'''
+                                         
+        transformed_data=pd.concat([data[features].shift(lag).add_suffix(f'_lag_{lag}') for lag in lags],axis=1)
+        transformed_data=transformed_data.fillna(method='bfill')
+
+        return pd.concat([transformed_data,data],axis=1)
+    
+#adstock
+def adstock(df, alphas, cutoff, features,dma=None):
+    
+    if dma:
+        transformed_data=pd.DataFrame()
+        for d in df[dma].unique():
+            dma_sub_df = df[df[dma] == d]
+            n = len(dma_sub_df)
+                                
+ 
+            weights = np.array([[[alpha**(i-j) if i >= j and j >= i-cutoff else 0. for j in range(n)] for i in range(n)] for alpha in alphas])
+
+            X = dma_sub_df[features].to_numpy()
+            res = pd.DataFrame(np.hstack(weights @ X), 
+                               columns=[f'{col}_adstock_{alpha}' for alpha in alphas for col in features])
+            
+            transformed_data=pd.concat([transformed_data,res],axis=0)
+            transformed_data.reset_index(drop=True,inplace=True)
+        return pd.concat([transformed_data,df],axis=1)
+
+    else:
+        
+        n = len(df)
+
+
+        weights = np.array([[[alpha**(i-j) if i >= j and j >= i-cutoff else 0. for j in range(n)] for i in range(n)] for alpha in alphas])
+
+        X = df[features].to_numpy()
+        res = pd.DataFrame(np.hstack(weights @ X), 
+                           columns=[f'{col}_adstock_{alpha}' for alpha in alphas for col in features])
+        return  pd.concat([res,df],axis=1)
+    
+
+
+
+
+if 'media_data' not in st.session_state:
+   
+  st.session_state['media_data']=pd.DataFrame()
+
+variables_to_be_transformed=[col for col in media_data.columns if col.lower() not in ['dma','panel'] ] # change for buckets
+
+
+if st.button('Apply Transformations'):
+   with st.spinner('Applying Transformations'):
+    transformed_data_lag=lag(media_data,features=variables_to_be_transformed,lags=np.arange(slider_value_lag[0],slider_value_lag[1]+1,1),dma=dma)
+
+    variables_to_be_transformed=[col for col in list(transformed_data_lag.columns) if col not in ['Date','DMA','Panel']] #change for buckets
+
+    transformed_data_adstock=adstock(df=transformed_data_lag, alphas=np.arange(slider_value_adstock[0],slider_value_adstock[1]+0.1,0.1), cutoff=8, features=variables_to_be_transformed,dma=dma)
+
+    st.success('Done')
+    st.write(f'old shape {old_shape}, new shape {transformed_data_adstock.shape}')
+    st.write(media_data.head(10))
+    st.write(transformed_data_adstock)
+    st.write(transformed_data_adstock.isnull().sum().sort_values(ascending=False))
+
+
+
+
+
+
+    
+#     st.write(dma_dict)
+#     st.session_state['media_data']=media_data
+
+#     with st.spinner('Applying Transformations'):
+#       time.sleep(2)
+#       st.success("Transformations complete!")
+
+# if st.session_state['media_data'].shape[1]>old_shape[1]:
+#   with columns[0]:
+#     st.write(f'Total no.of variables before transformation: {old_shape[1]}, Total no.of variables after transformation: {st.session_state["media_data"].shape[1]}')
+#   #st.write(f'Total no.of variables after transformation: {st.session_state["media_data"].shape[1]}')
+
+
+# bucket=['paid_search', 'kwai','indicacao','infleux', 'influencer','FB: Level Achieved - Tier 1 Impressions',
+#       ' FB: Level Achieved - Tier 2 Impressions','paid_social_others',
+#         ' GA App: Will And Cid Pequena Baixo Risco Clicks',
+#       'digital_tactic_others',"programmatic"
+#       ]
+   
+# with columns[1]:
+#   if st.button('Create Combinations of Variables'):
+
+#     top_3_correlated_features=[]
+#     for col in st.session_state['media_data'].columns[:19]:
+#         corr_df=pd.concat([st.session_state['media_data'].filter(regex=col),
+#                   revenue],axis=1).corr()['Total Approved Accounts - Revenue'].iloc[:-1]
+#         top_3_correlated_features.append(list(corr_df.sort_values(ascending=False).head(2).index))
+#     flattened_list = [item for sublist in top_3_correlated_features for item in sublist]
+#     all_features_set={var:[col for col in flattened_list if var in col] for var in bucket}
+#     channels_all=[values for values in all_features_set.values()]
+#     st.session_state['combinations'] = list(itertools.product(*channels_all))
+
+#   # if 'combinations' not in st.session_state:  
+#   #   st.session_state['combinations']=combinations_all
+
+#     st.session_state['final_selection']=st.session_state['combinations']
+
+
+
+# revenue.reset_index(drop=True,inplace=True)
+# if 'Model_results' not in st.session_state:
+#       st.session_state['Model_results']={'Model_object':[],
+#     'Model_iteration':[],
+#     'Feature_set':[],
+#     'MAPE':[],
+#     'R2':[],
+#     'ADJR2':[]
+#     }
+
+# #if st.button('Build Model'):
+# if 'iterations' not in st.session_state:
+#    st.session_state['iterations']=1
+# save_path = r"Model"
+# with columns[1]:
+#   if "final_selection" in st.session_state:
+#     st.write(f'Total combinations created {format_numbers(len(st.session_state["final_selection"]))}')
+    
+#     st.success('Done')
+# if st.checkbox('Build all iterations'):
+#    iterations=len(st.session_state['final_selection'])
+# else:
+#    iterations = st.number_input('Select the number of iterations to perform', min_value=1, step=100, value=st.session_state['iterations'])  
+
+# st.session_state['iterations']=iterations
+
+
+# st.session_state['media_data']=st.session_state['media_data'].fillna(method='ffill')
+# if st.button("Build Models"):  
+#   st.markdown('Data Split -- Training Period: May 9th, 2023 - October 5th,2023 , Testing Period: October 6th, 2023 - November 7th, 2023 ')
+#   progress_bar = st.progress(0)  # Initialize the progress bar
+#   #time_remaining_text = st.empty()  # Create an empty space for time remaining text
+#   start_time = time.time()  # Record the start time
+#   progress_text = st.empty()
+#   #time_elapsed_text = st.empty()
+#   for i, selected_features in enumerate(st.session_state["final_selection"][40000:40000+int(iterations)]):
+#       df = st.session_state['media_data']
+
+#       fet = [var for var in selected_features if len(var) > 0]
+#       X = df[fet]
+#       y = revenue
+#       ss = MinMaxScaler()
+#       X = pd.DataFrame(ss.fit_transform(X), columns=X.columns)
+#       X = sm.add_constant(X)
+#       X_train=X.iloc[:150]
+#       X_test=X.iloc[150:]
+#       y_train=y.iloc[:150]
+#       y_test=y.iloc[150:]
+
+
+#       model = sm.OLS(y_train, X_train).fit()
+#       # st.write(fet)
+#       positive_coeff=X.columns
+#       negetive_coeff=[]
+#       coefficients=model.params.to_dict()
+#       model_possitive=[col for col in coefficients.keys() if coefficients[col]>0]
+#       # st.write(positive_coeff)
+#       # st.write(model_possitive)
+#       pvalues=[var for var in list(model.pvalues) if var<=0.06]
+#       if (len(model_possitive)/len(selected_features))>0.9 and (len(pvalues)/len(selected_features))>=0.8:
+
+
+#           predicted_values = model.predict(X_train)
+#           mape = mean_absolute_percentage_error(y_train, predicted_values)
+#           adjr2 = model.rsquared_adj
+#           r2 = model.rsquared
+#           filename = os.path.join(save_path, f"model_{i}.pkl")
+#           with open(filename, "wb") as f:
+#             pickle.dump(model, f)
+#           # with open(r"C:\Users\ManojP\Documents\MMM\simopt\Model\model.pkl", 'rb') as file:
+#           #   model = pickle.load(file)
+
+#           st.session_state['Model_results']['Model_object'].append(filename)
+#           st.session_state['Model_results']['Model_iteration'].append(i)
+#           st.session_state['Model_results']['Feature_set'].append(fet)
+#           st.session_state['Model_results']['MAPE'].append(mape)
+#           st.session_state['Model_results']['R2'].append(r2)
+#           st.session_state['Model_results']['ADJR2'].append(adjr2)
+
+#       current_time = time.time()
+#       time_taken = current_time - start_time
+#       time_elapsed_minutes = time_taken / 60
+#       completed_iterations_text = f"{i + 1}/{iterations}"
+#       progress_bar.progress((i + 1) / int(iterations))
+#       progress_text.text(f'Completed iterations: {completed_iterations_text},Time Elapsed (min): {time_elapsed_minutes:.2f}')
+  
+# st.write(f'Out of {st.session_state["iterations"]} iterations : {len(st.session_state["Model_results"]["Model_object"])} valid models')
+# pd.DataFrame(st.session_state['Model_results']).to_csv('model_output.csv')
+
+# def to_percentage(value):
+#   return f'{value * 100:.1f}%'   
+
+# st.title('2. Select Models')
+# if 'tick' not in st.session_state:
+#    st.session_state['tick']=False
+# if st.checkbox('Show results of top 10 models (based on MAPE and Adj. R2)',value=st.session_state['tick']):
+#   st.session_state['tick']=True
+#   st.write('Select one model iteration to generate performance metrics for it:')
+#   data=pd.DataFrame(st.session_state['Model_results'])
+#   data.sort_values(by=['MAPE'],ascending=False,inplace=True)
+#   data.drop_duplicates(subset='Model_iteration',inplace=True)
+#   top_10=data.head(10)
+#   top_10['Rank']=np.arange(1,len(top_10)+1,1)
+#   top_10[['MAPE','R2','ADJR2']]=np.round(top_10[['MAPE','R2','ADJR2']],4).applymap(to_percentage)
+#   top_10_table = top_10[['Rank','Model_iteration','MAPE','ADJR2','R2']]
+#   #top_10_table.columns=[['Rank','Model Iteration Index','MAPE','Adjusted R2','R2']]
+#   gd=GridOptionsBuilder.from_dataframe(top_10_table)
+#   gd.configure_pagination(enabled=True)
+#   gd.configure_selection(use_checkbox=True)
+
+  
+#   gridoptions=gd.build()
+
+#   table = AgGrid(top_10,gridOptions=gridoptions,update_mode=GridUpdateMode.SELECTION_CHANGED)
+  
+#   selected_rows=table.selected_rows
+#   # if st.session_state["selected_rows"] != selected_rows:
+#   #   st.session_state["build_rc_cb"] = False
+#   st.session_state["selected_rows"] = selected_rows
+#   if 'Model' not in st.session_state:
+#     st.session_state['Model']={}
+
+#   if len(selected_rows)>0:
+#     st.header('2.1 Results Summary')
+
+#     model_object=data[data['Model_iteration']==selected_rows[0]['Model_iteration']]['Model_object']
+#     features_set=data[data['Model_iteration']==selected_rows[0]['Model_iteration']]['Feature_set']
+    
+#     with open(str(model_object.values[0]), 'rb') as file:
+#             model = pickle.load(file)
+#     st.write(model.summary())        
+#     st.header('2.2 Actual vs. Predicted Plot')
+  
+#     df=st.session_state['media_data']
+#     X=df[features_set.values[0]]
+#     X = sm.add_constant(X)
+#     y=revenue
+#     X_train=X.iloc[:150]
+#     X_test=X.iloc[150:]
+#     y_train=y.iloc[:150]
+#     y_test=y.iloc[150:]
+#     ss = MinMaxScaler()
+#     X_train = pd.DataFrame(ss.fit_transform(X_train), columns=X_train.columns)
+#     st.session_state['X']=X_train
+#     st.session_state['features_set']=features_set.values[0]
+
+#     metrics_table,line,actual_vs_predicted_plot=plot_actual_vs_predicted(date, y_train, model.predict(X_train), model,target_column='Revenue')
+
+#     st.plotly_chart(actual_vs_predicted_plot,use_container_width=True)
+
+
+     
+#     st.markdown('## 2.3 Residual Analysis')
+#     columns=st.columns(2)
+#     with columns[0]:
+#       fig=plot_residual_predicted(y_train,model.predict(X_train),X_train)
+#       st.plotly_chart(fig)
+    
+#     with columns[1]:
+#       st.empty()
+#       fig = qqplot(y_train,model.predict(X_train))
+#       st.plotly_chart(fig)
+
+#     with columns[0]:
+#       fig=residual_distribution(y_train,model.predict(X_train))
+#       st.pyplot(fig)
+    
+
+
+#     vif_data = pd.DataFrame()
+#     # X=X.drop('const',axis=1)
+#     vif_data["Variable"] = X_train.columns
+#     vif_data["VIF"] = [variance_inflation_factor(X_train.values, i) for i in range(X_train.shape[1])]
+#     vif_data.sort_values(by=['VIF'],ascending=False,inplace=True)
+#     vif_data=np.round(vif_data)
+#     vif_data['VIF']=vif_data['VIF'].astype(float)
+#     st.header('2.4 Variance Inflation Factor (VIF)')
+#     #st.dataframe(vif_data)
+#     color_mapping = {
+#     'darkgreen': (vif_data['VIF'] < 3),
+#     'orange': (vif_data['VIF'] >= 3) & (vif_data['VIF'] <= 10),
+#     'darkred': (vif_data['VIF'] > 10)
+#     }
+
+# # Create a horizontal bar plot
+#     fig, ax = plt.subplots()
+#     fig.set_figwidth(10)  # Adjust the width of the figure as needed
+
+#     # Sort the bars by descending VIF values
+#     vif_data = vif_data.sort_values(by='VIF', ascending=False)
+
+#     # Iterate through the color mapping and plot bars with corresponding colors
+#     for color, condition in color_mapping.items():
+#         subset = vif_data[condition]
+#         bars = ax.barh(subset["Variable"], subset["VIF"], color=color, label=color)
+        
+#         # Add text annotations on top of the bars
+#         for bar in bars:
+#             width = bar.get_width()
+#             ax.annotate(f'{width:}', xy=(width, bar.get_y() + bar.get_height() / 2), xytext=(5, 0),
+#                         textcoords='offset points', va='center')
+
+#     # Customize the plot
+#     ax.set_xlabel('VIF Values')
+#     #ax.set_title('2.4 Variance Inflation Factor (VIF)')
+#     #ax.legend(loc='upper right')
+
+#     # Display the plot in Streamlit
+#     st.pyplot(fig)
+
+#     with st.expander('Results Summary Test data'):
+#       ss = MinMaxScaler()
+#       X_test = pd.DataFrame(ss.fit_transform(X_test), columns=X_test.columns)
+#       st.header('2.2 Actual vs. Predicted Plot')
+
+#       metrics_table,line,actual_vs_predicted_plot=plot_actual_vs_predicted(date, y_test, model.predict(X_test), model,target_column='Revenue')
+
+#       st.plotly_chart(actual_vs_predicted_plot,use_container_width=True)
+
+#       st.markdown('## 2.3 Residual Analysis')
+#       columns=st.columns(2)
+#       with columns[0]:
+#         fig=plot_residual_predicted(revenue,model.predict(X_test),X_test)
+#         st.plotly_chart(fig)
+      
+#       with columns[1]:
+#         st.empty()
+#         fig = qqplot(revenue,model.predict(X_test))
+#         st.plotly_chart(fig)
+
+#       with columns[0]:
+#         fig=residual_distribution(revenue,model.predict(X_test))
+#         st.pyplot(fig)
+        
+#     value=False
+#     if st.checkbox('Save this model to tune',key='build_rc_cb'):
+#       mod_name=st.text_input('Enter model name')
+#       if len(mod_name)>0:
+#         st.session_state['Model'][mod_name]={"Model_object":model,'feature_set':st.session_state['features_set'],'X_train':X_train}
+#         st.session_state['X_train']=X_train
+#         st.session_state['X_test']=X_test
+#         st.session_state['y_train']=y_train
+#         st.session_state['y_test']=y_test
+#         with open("best_models.pkl", "wb") as f:
+#           pickle.dump(st.session_state['Model'], f)  
+#           st.success('Model saved!, Proceed  next page to tune the model')
+#         value=False

dump/1_Transformations_with_panel.py

@@ -0,0 +1,548 @@
+import streamlit as st
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+from Eda_functions import format_numbers
+import numpy as np
+import pickle
+from st_aggrid import AgGrid
+from st_aggrid import GridOptionsBuilder,GridUpdateMode
+from utilities import set_header,load_local_css
+from st_aggrid import GridOptionsBuilder
+import time
+import itertools
+import statsmodels.api as sm
+import numpy as npc
+import re
+import itertools
+from sklearn.metrics import mean_absolute_error, r2_score,mean_absolute_percentage_error
+from sklearn.preprocessing import MinMaxScaler
+import os
+import matplotlib.pyplot as plt
+from statsmodels.stats.outliers_influence import variance_inflation_factor
+st.set_option('deprecation.showPyplotGlobalUse', False)
+import statsmodels.api as sm
+import statsmodels.formula.api as smf
+
+from datetime import datetime
+import seaborn as sns
+from Data_prep_functions import *
+
+
+def get_random_effects(media_data, panel_col, mdf):
+    random_eff_df = pd.DataFrame(columns=[panel_col, "random_effect"])
+
+    for i, market in enumerate(media_data[panel_col].unique()):
+        print(i, end='\r')
+        intercept = mdf.random_effects[market].values[0]
+        random_eff_df.loc[i, 'random_effect'] = intercept
+        random_eff_df.loc[i, panel_col] = market
+
+    return random_eff_df
+
+
+def mdf_predict(X, mdf, random_eff_df) :
+
+    X['fixed_effect'] = mdf.predict(X)
+    merged_df=pd.merge(X[[panel_col,target_col]], random_eff_df, on = panel_col, how = 'left')
+    X['random_effect'] = merged_df['random_effect']
+    X['pred'] = X['fixed_effect'] + X['random_effect']
+    return X['pred']
+
+st.set_page_config(
+  page_title="Model Build",
+  page_icon=":shark:",
+  layout="wide",
+  initial_sidebar_state='collapsed'
+)
+
+load_local_css('styles.css')
+set_header()
+
+
+st.title('1. Build Your Model')
+
+panel_col= 'markets' # set the panel column
+date_col = 'date'
+target_col = 'total_approved_accounts_revenue'
+
+media_data=pd.read_csv('upf_data_converted.csv')
+media_data.columns=[i.lower().replace('-','').replace(':','').replace("__", "_") for i in media_data.columns]
+
+# st.write(media_data.columns)
+media_data.sort_values(date_col, inplace=True)
+media_data.reset_index(drop=True,inplace=True)
+
+date=media_data[date_col]
+st.session_state['date']=date
+revenue=media_data[target_col]
+media_data.drop([target_col],axis=1,inplace=True)
+media_data.drop([date_col],axis=1,inplace=True)
+media_data.reset_index(drop=True,inplace=True)
+
+
+if st.toggle('Apply Transformations on DMA/Panel Level'):
+  dma=st.selectbox('Select the Level of data ',[ col for col in media_data.columns if col.lower() in ['dma','panel', 'markets']])
+
+
+else:
+  #""" code to aggregate data on date """
+
+
+  dma=None
+
+# dma_dict={ dm:media_data[media_data[dma]==dm] for dm in media_data[dma].unique()}
+# st.write(dma_dict)
+
+st.markdown('## Select the Range of Transformations')
+columns = st.columns(2)
+old_shape=media_data.shape
+
+
+if "old_shape" not in st.session_state:
+   st.session_state['old_shape']=old_shape
+
+
+with columns[0]:
+  slider_value_adstock  = st.slider('Select Adstock Range (only applied to media)', 0.0, 1.0, (0.2, 0.4), step=0.1, format="%.2f")
+with columns[1]:
+  slider_value_lag = st.slider('Select Lag Range (applied to media, seasonal, macroeconomic variables)', 1, 7, (1, 3), step=1)
+
+# with columns[2]:
+#    slider_value_power=st.slider('Select Power range (only applied to media )',0,4,(1,2),step=1)
+
+# with columns[1]:
+#    st.number_input('Select the range of half saturation point ',min_value=1,max_value=5)
+#    st.number_input('Select the range of  ')
+
+# Section 1 - Transformations Functions
+def lag(data,features,lags,dma=None):
+    if dma:
+
+        transformed_data=pd.concat([data.groupby([dma])[features].shift(lag).add_suffix(f'_lag_{lag}') for lag in lags],axis=1)
+        transformed_data=transformed_data.fillna(method='bfill')
+        return pd.concat([transformed_data,data],axis=1)
+
+    else:
+
+        #''' data should be aggregated on date'''
+
+        transformed_data=pd.concat([data[features].shift(lag).add_suffix(f'_lag_{lag}') for lag in lags],axis=1)
+        transformed_data=transformed_data.fillna(method='bfill')
+
+        return pd.concat([transformed_data,data],axis=1)
+
+#adstock
+def adstock(df, alphas, cutoff, features,dma=None):
+    # st.write(features)
+
+    if dma:
+        transformed_data=pd.DataFrame()
+        for d in df[dma].unique():
+            dma_sub_df = df[df[dma] == d]
+            n = len(dma_sub_df)
+
+
+            weights = np.array([[[alpha**(i-j) if i >= j and j >= i-cutoff else 0. for j in range(n)] for i in range(n)] for alpha in alphas])
+            X = dma_sub_df[features].to_numpy()
+
+            res = pd.DataFrame(np.hstack(weights @ X),
+                               columns=[f'{col}_adstock_{alpha}' for alpha in alphas for col in features])
+
+            transformed_data=pd.concat([transformed_data,res],axis=0)
+            transformed_data.reset_index(drop=True,inplace=True)
+        return pd.concat([transformed_data,df],axis=1)
+
+    else:
+
+        n = len(df)
+
+
+        weights = np.array([[[alpha**(i-j) if i >= j and j >= i-cutoff else 0. for j in range(n)] for i in range(n)] for alpha in alphas])
+
+        X = df[features].to_numpy()
+        res = pd.DataFrame(np.hstack(weights @ X),
+                           columns=[f'{col}_adstock_{alpha}' for alpha in alphas for col in features])
+        return  pd.concat([res,df],axis=1)
+
+
+
+
+# Section 2 - Begin Transformations
+
+if 'media_data' not in st.session_state:
+
+  st.session_state['media_data']=pd.DataFrame()
+
+# variables_to_be_transformed=[col for col in media_data.columns if col.lower() not in ['dma','panel'] ] # change for buckets
+variables_to_be_transformed=[col for col in media_data.columns if '_clicks' in col.lower() or '_impress' in col.lower()] # srishti - change
+# st.write(variables_to_be_transformed)
+# st.write(media_data[variables_to_be_transformed].dtypes)
+
+with columns[0]:
+  if st.button('Apply Transformations'):
+    with st.spinner('Applying Transformations'):
+      transformed_data_lag=lag(media_data,features=variables_to_be_transformed,lags=np.arange(slider_value_lag[0],slider_value_lag[1]+1,1),dma=dma)
+
+      # variables_to_be_transformed=[col for col in list(transformed_data_lag.columns) if col not in ['Date','DMA','Panel']] #change for buckets
+      variables_to_be_transformed = [col for col in media_data.columns if
+                                    '_clicks' in col.lower() or '_impress' in col.lower()]  # srishti - change
+
+      transformed_data_adstock=adstock(df=transformed_data_lag, alphas=np.arange(slider_value_adstock[0],slider_value_adstock[1],0.1), cutoff=8, features=variables_to_be_transformed,dma=dma)
+
+      # st.success('Done')
+      st.success("Transformations complete!")
+
+      st.write(f'old shape {old_shape}, new shape {transformed_data_adstock.shape}')
+      # st.write(media_data.head(10))
+      # st.write(transformed_data_adstock.head(10))
+
+      transformed_data_adstock.columns = [c.replace(".","_") for c in transformed_data_adstock.columns] # srishti
+      # st.write(transformed_data_adstock.columns)
+      st.session_state['media_data']=transformed_data_adstock # srishti
+
+    # with st.spinner('Applying Transformations'):
+    #   time.sleep(2)
+    #   st.success("Transformations complete!")
+
+# if st.session_state['media_data'].shape[1]>old_shape[1]:
+  # with columns[0]:
+    # st.write(f'Total no.of variables before transformation: {old_shape[1]}, Total no.of variables after transformation: {st.session_state["media_data"].shape[1]}')
+  #st.write(f'Total no.of variables after transformation: {st.session_state["media_data"].shape[1]}')
+
+# Section 3 - Create combinations
+
+# bucket=['paid_search', 'kwai','indicacao','infleux', 'influencer','FB: Level Achieved - Tier 1 Impressions',
+#       ' FB: Level Achieved - Tier 2 Impressions','paid_social_others',
+#         ' GA App: Will And Cid Pequena Baixo Risco Clicks',
+#       'digital_tactic_others',"programmatic"
+#       ]
+
+# srishti - bucket names changed
+bucket=['paid_search', 'kwai','indicacao','infleux', 'influencer','fb_level_achieved_tier_2',
+      'fb_level_achieved_tier_1','paid_social_others',
+        'ga_app',
+      'digital_tactic_others',"programmatic"
+      ]
+
+with columns[1]:
+  if st.button('Create Combinations of Variables'):
+
+    top_3_correlated_features=[]
+    # for col in st.session_state['media_data'].columns[:19]:
+    original_cols = [c for c in st.session_state['media_data'].columns if "_clicks" in c.lower() or "_impressions" in c.lower()]
+    original_cols = [c for c in original_cols if "_lag" not in c.lower() and "_adstock" not in c.lower()]
+    # st.write(original_cols)
+
+    # for col in st.session_state['media_data'].columns[:19]:
+    for col in original_cols: # srishti - new
+        corr_df=pd.concat([st.session_state['media_data'].filter(regex=col),
+                  revenue],axis=1).corr()[target_col].iloc[:-1]
+        top_3_correlated_features.append(list(corr_df.sort_values(ascending=False).head(2).index))
+        # st.write(col, top_3_correlated_features)
+    flattened_list = [item for sublist in top_3_correlated_features for item in sublist]
+    # all_features_set={var:[col for col in flattened_list if var in col] for var in bucket}
+    all_features_set={var:[col for col in flattened_list if var in col] for var in bucket if len([col for col in flattened_list if var in col])>0} # srishti
+
+    channels_all=[values for values in all_features_set.values()]
+    # st.write(channels_all)
+    st.session_state['combinations'] = list(itertools.product(*channels_all))
+  # if 'combinations' not in st.session_state:
+  #   st.session_state['combinations']=combinations_all
+
+    st.session_state['final_selection']=st.session_state['combinations']
+    st.success('Done')
+    # st.write(f"{len(st.session_state['combinations'])} combinations created")
+
+
+    revenue.reset_index(drop=True,inplace=True)
+  if 'Model_results' not in st.session_state:
+        st.session_state['Model_results']={'Model_object':[],
+      'Model_iteration':[],
+      'Feature_set':[],
+      'MAPE':[],
+      'R2':[],
+      'ADJR2':[]
+      }
+
+  def reset_model_result_dct():
+      st.session_state['Model_results']={'Model_object':[],
+      'Model_iteration':[],
+      'Feature_set':[],
+      'MAPE':[],
+      'R2':[],
+      'ADJR2':[]
+      }
+
+      # if st.button('Build Model'):
+  if 'iterations' not in st.session_state:
+    st.session_state['iterations']=0
+      # st.write("1",st.session_state["final_selection"])
+
+  if 'final_selection' not in st.session_state:
+      st.session_state['final_selection']=False
+
+save_path = r"Model/"
+with columns[1]:
+  if  st.session_state['final_selection']:
+    st.write(f'Total combinations created {format_numbers(len(st.session_state["final_selection"]))}')
+
+    
+if st.checkbox('Build all iterations'):
+   iterations=len(st.session_state['final_selection'])
+else:
+   iterations = st.number_input('Select the number of iterations to perform', min_value=0, step=10, value=st.session_state['iterations'],on_change=reset_model_result_dct)
+  #  st.write("iterations=", iterations)
+
+if st.button('Build Model',on_click=reset_model_result_dct):
+  st.session_state['iterations']=iterations 
+  # st.write("2",st.session_state["final_selection"])
+
+  # Section 4 - Model
+
+  st.session_state['media_data']=st.session_state['media_data'].fillna(method='ffill')
+  st.markdown(
+      'Data Split -- Training Period: May 9th, 2023 - October 5th,2023 , Testing Period: October 6th, 2023 - November 7th, 2023 ')
+  progress_bar = st.progress(0)  # Initialize the progress bar
+  # time_remaining_text = st.empty()  # Create an empty space for time remaining text
+  start_time = time.time()  # Record the start time
+  progress_text = st.empty()
+  # time_elapsed_text = st.empty()
+  # for i, selected_features in enumerate(st.session_state["final_selection"][40000:40000 + int(iterations)]):
+  # st.write(st.session_state["final_selection"])
+  # for i, selected_features in enumerate(st.session_state["final_selection"]):
+  for i, selected_features in enumerate(st.session_state["final_selection"][0:int(iterations)]): # srishti
+      print("@@@@@@@@@@@@@",i)
+      df = st.session_state['media_data']
+
+      fet = [var for var in selected_features if len(var) > 0]
+      inp_vars_str = " + ".join(fet)  # new
+
+      X = df[fet]
+      y = revenue
+      ss = MinMaxScaler()
+      X = pd.DataFrame(ss.fit_transform(X), columns=X.columns)
+      # X = sm.add_constant(X)
+
+      X['total_approved_accounts_revenue'] = revenue  # new
+      X[panel_col] = df[panel_col]
+
+      X_train = X.iloc[:8000]
+      X_test = X.iloc[8000:]
+      y_train = y.iloc[:8000]
+      y_test = y.iloc[8000:]
+
+      print(X_train.shape)
+      # model = sm.OLS(y_train, X_train).fit()
+      md = smf.mixedlm("total_approved_accounts_revenue ~ {}".format(inp_vars_str),
+                      data=X_train[['total_approved_accounts_revenue'] + fet],
+                      groups=X_train[panel_col])
+      mdf = md.fit()
+      predicted_values = mdf.fittedvalues
+
+      # st.write(fet)
+      # positive_coeff=fet
+      # negetive_coeff=[]
+
+      coefficients = mdf.fe_params.to_dict()
+      model_possitive = [col for col in coefficients.keys() if coefficients[col] > 0]
+      # st.write(positive_coeff)
+      # st.write(model_possitive)
+      pvalues = [var for var in list(mdf.pvalues) if var <= 0.06]
+
+      # if (len(model_possitive) / len(selected_features)) > 0.9 and (len(pvalues) / len(selected_features)) >= 0.8:
+      if (len(model_possitive) / len(selected_features)) > 0 and (len(pvalues) / len(selected_features)) >= 0: # srishti - changed just for testing, revert later
+          # predicted_values = model.predict(X_train)
+          mape = mean_absolute_percentage_error(y_train, predicted_values)
+          r2 = r2_score(y_train, predicted_values)
+          adjr2 = 1 - (1 - r2) * (len(y_train) - 1) / (len(y_train) - len(selected_features) - 1)
+
+          filename = os.path.join(save_path, f"model_{i}.pkl")
+          with open(filename, "wb") as f:
+              pickle.dump(mdf, f)
+          # with open(r"C:\Users\ManojP\Documents\MMM\simopt\Model\model.pkl", 'rb') as file:
+          #   model = pickle.load(file)
+
+          st.session_state['Model_results']['Model_object'].append(filename)
+          st.session_state['Model_results']['Model_iteration'].append(i)
+          st.session_state['Model_results']['Feature_set'].append(fet)
+          st.session_state['Model_results']['MAPE'].append(mape)
+          st.session_state['Model_results']['R2'].append(r2)
+          st.session_state['Model_results']['ADJR2'].append(adjr2)
+
+      current_time = time.time()
+      time_taken = current_time - start_time
+      time_elapsed_minutes = time_taken / 60
+      completed_iterations_text = f"{i + 1}/{iterations}"
+      progress_bar.progress((i + 1) / int(iterations))
+      progress_text.text(f'Completed iterations: {completed_iterations_text},Time Elapsed (min): {time_elapsed_minutes:.2f}')
+
+  st.write(f'Out of {st.session_state["iterations"]} iterations : {len(st.session_state["Model_results"]["Model_object"])} valid models')
+  pd.DataFrame(st.session_state['Model_results']).to_csv('model_output.csv')
+
+  def to_percentage(value):
+    return f'{value * 100:.1f}%'
+
+st.title('2. Select Models')
+if 'tick' not in st.session_state:
+   st.session_state['tick']=False
+if st.checkbox('Show results of top 10 models (based on MAPE and Adj. R2)',value=st.session_state['tick']):
+  st.session_state['tick']=True
+  st.write('Select one model iteration to generate performance metrics for it:')
+  data=pd.DataFrame(st.session_state['Model_results'])
+  data.sort_values(by=['MAPE'],ascending=False,inplace=True)
+  data.drop_duplicates(subset='Model_iteration',inplace=True)
+  top_10=data.head(10)
+  top_10['Rank']=np.arange(1,len(top_10)+1,1)
+  top_10[['MAPE','R2','ADJR2']]=np.round(top_10[['MAPE','R2','ADJR2']],4).applymap(to_percentage)
+  top_10_table = top_10[['Rank','Model_iteration','MAPE','ADJR2','R2']]
+  #top_10_table.columns=[['Rank','Model Iteration Index','MAPE','Adjusted R2','R2']]
+  gd=GridOptionsBuilder.from_dataframe(top_10_table)
+  gd.configure_pagination(enabled=True)
+  gd.configure_selection(use_checkbox=True)
+
+
+  gridoptions=gd.build()
+
+  table = AgGrid(top_10,gridOptions=gridoptions,update_mode=GridUpdateMode.SELECTION_CHANGED)
+
+  selected_rows=table.selected_rows
+  # if st.session_state["selected_rows"] != selected_rows:
+  #   st.session_state["build_rc_cb"] = False
+  st.session_state["selected_rows"] = selected_rows
+  if 'Model' not in st.session_state:
+    st.session_state['Model']={}
+
+  if len(selected_rows)>0:
+    st.header('2.1 Results Summary')
+
+    model_object=data[data['Model_iteration']==selected_rows[0]['Model_iteration']]['Model_object']
+    features_set=data[data['Model_iteration']==selected_rows[0]['Model_iteration']]['Feature_set']
+
+    with open(str(model_object.values[0]), 'rb') as file:
+        # print(file)
+        model = pickle.load(file)
+    st.write(model.summary())
+    st.header('2.2 Actual vs. Predicted Plot')
+
+    df=st.session_state['media_data']
+    X=df[features_set.values[0]]
+    # X = sm.add_constant(X)
+    y=revenue
+
+    ss = MinMaxScaler()
+    X = pd.DataFrame(ss.fit_transform(X), columns=X.columns)
+
+    X['total_approved_accounts_revenue'] = revenue  # new
+    X[panel_col] = df[panel_col]
+
+    X_train=X.iloc[:8000]
+    X_test=X.iloc[8000:]
+    y_train=y.iloc[:8000]
+    y_test=y.iloc[8000:]
+
+    st.session_state['X']=X_train
+    st.session_state['features_set']=features_set.values[0]
+
+    metrics_table,line,actual_vs_predicted_plot=plot_actual_vs_predicted(date, y_train, model.fittedvalues, model,target_column='Revenue')
+
+    st.plotly_chart(actual_vs_predicted_plot,use_container_width=True)
+
+    random_eff_df = get_random_effects(media_data, panel_col, model)
+
+
+    st.markdown('## 2.3 Residual Analysis')
+    columns=st.columns(2)
+    with columns[0]:
+      fig=plot_residual_predicted(y_train,model.fittedvalues,X_train)
+      st.plotly_chart(fig)
+
+    with columns[1]:
+      st.empty()
+      fig = qqplot(y_train,model.fittedvalues)
+      st.plotly_chart(fig)
+
+    with columns[0]:
+      fig=residual_distribution(y_train,model.fittedvalues)
+      st.pyplot(fig)
+
+
+
+    vif_data = pd.DataFrame()
+    # X=X.drop('const',axis=1)
+    vif_data["Variable"] = X_train.columns
+    vif_data["VIF"] = [variance_inflation_factor(X_train.values, i) for i in range(X_train.shape[1])]
+    vif_data.sort_values(by=['VIF'],ascending=False,inplace=True)
+    vif_data=np.round(vif_data)
+    vif_data['VIF']=vif_data['VIF'].astype(float)
+    st.header('2.4 Variance Inflation Factor (VIF)')
+    #st.dataframe(vif_data)
+    color_mapping = {
+    'darkgreen': (vif_data['VIF'] < 3),
+    'orange': (vif_data['VIF'] >= 3) & (vif_data['VIF'] <= 10),
+    'darkred': (vif_data['VIF'] > 10)
+    }
+
+# Create a horizontal bar plot
+    fig, ax = plt.subplots()
+    fig.set_figwidth(10)  # Adjust the width of the figure as needed
+
+    # Sort the bars by descending VIF values
+    vif_data = vif_data.sort_values(by='VIF', ascending=False)
+
+    # Iterate through the color mapping and plot bars with corresponding colors
+    for color, condition in color_mapping.items():
+        subset = vif_data[condition]
+        bars = ax.barh(subset["Variable"], subset["VIF"], color=color, label=color)
+
+        # Add text annotations on top of the bars
+        for bar in bars:
+            width = bar.get_width()
+            ax.annotate(f'{width:}', xy=(width, bar.get_y() + bar.get_height() / 2), xytext=(5, 0),
+                        textcoords='offset points', va='center')
+
+    # Customize the plot
+    ax.set_xlabel('VIF Values')
+    #ax.set_title('2.4 Variance Inflation Factor (VIF)')
+    #ax.legend(loc='upper right')
+
+    # Display the plot in Streamlit
+    st.pyplot(fig)
+
+    with st.expander('Results Summary Test data'):
+      ss = MinMaxScaler()
+      X_test = pd.DataFrame(ss.fit_transform(X_test), columns=X_test.columns)
+      st.header('2.2 Actual vs. Predicted Plot')
+
+      metrics_table,line,actual_vs_predicted_plot=plot_actual_vs_predicted(date, y_test, mdf_predict(X_test,mdf, random_eff_df), model,target_column='Revenue')
+
+      st.plotly_chart(actual_vs_predicted_plot,use_container_width=True)
+
+      st.markdown('## 2.3 Residual Analysis')
+      columns=st.columns(2)
+      with columns[0]:
+        fig=plot_residual_predicted(revenue,mdf_predict(X_test,mdf, random_eff_df),X_test)
+        st.plotly_chart(fig)
+
+      with columns[1]:
+        st.empty()
+        fig = qqplot(revenue,mdf_predict(X_test,mdf, random_eff_df))
+        st.plotly_chart(fig)
+
+      with columns[0]:
+        fig=residual_distribution(revenue,mdf_predict(X_test,mdf, random_eff_df))
+        st.pyplot(fig)
+
+    value=False
+    if st.checkbox('Save this model to tune',key='build_rc_cb'):
+      mod_name=st.text_input('Enter model name')
+      if len(mod_name)>0:
+        st.session_state['Model'][mod_name]={"Model_object":model,'feature_set':st.session_state['features_set'],'X_train':X_train}
+        st.session_state['X_train']=X_train
+        st.session_state['X_test']=X_test
+        st.session_state['y_train']=y_train
+        st.session_state['y_test']=y_test
+        with open("best_models.pkl", "wb") as f:
+          pickle.dump(st.session_state['Model'], f)
+          st.success('Model saved!, Proceed  next page to tune the model')
+        value=False

dump/2_Model_Build_and_Performance.py

@@ -0,0 +1,403 @@
+import streamlit as st
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+from Eda_functions import format_numbers
+import numpy as np
+import pickle
+from st_aggrid import AgGrid
+from st_aggrid import GridOptionsBuilder,GridUpdateMode
+from utilities import set_header,load_local_css
+from st_aggrid import GridOptionsBuilder
+import time
+import itertools
+import statsmodels.api as sm
+import numpy as np
+import re
+import itertools
+from sklearn.metrics import mean_absolute_error, r2_score,mean_absolute_percentage_error  
+from sklearn.preprocessing import MinMaxScaler
+import os
+import matplotlib.pyplot as plt
+from statsmodels.stats.outliers_influence import variance_inflation_factor
+st.set_option('deprecation.showPyplotGlobalUse', False)
+from datetime import datetime
+import seaborn as sns
+from Data_prep_functions import *
+
+st.set_page_config(
+  page_title="Model Build",
+  page_icon=":shark:",
+  layout="wide",
+  initial_sidebar_state='collapsed'
+)
+
+load_local_css('styles.css')
+set_header()
+
+
+st.title('1. Build Your Model')
+
+# media_data=pd.read_csv('Media_data_for_model.csv')
+media_data=pd.read_csv('Media_data_for_model_dma_level.csv')
+date=media_data['Date']
+st.session_state['date']=date
+revenue=media_data['Total Approved Accounts - Revenue']
+media_data.drop(['Total Approved Accounts - Revenue'],axis=1,inplace=True)
+media_data.drop(['Date'],axis=1,inplace=True)
+media_data.reset_index(drop=True,inplace=True)
+
+dma=st.selectbox('Select the Level of data ',[ col for col in media_data.columns if col.lower() in ['dma','panel']])
+
+dma_dict={ dm:media_data[media_data[dma]==dm] for dm in media_data[dma].unique()}
+# st.write(dma_dict)
+
+st.markdown('## Select the Range of Transformations')
+columns = st.columns(2)
+old_shape=media_data.shape
+
+
+
+
+
+if "old_shape" not in st.session_state:
+   st.session_state['old_shape']=old_shape
+
+
+
+
+with columns[0]:
+  slider_value_adstock  = st.slider('Select Adstock Range (only applied to media)', 0.0, 1.0, (0.2, 0.4), step=0.1, format="%.2f")
+with columns[1]:
+  slider_value_lag = st.slider('Select Lag Range (applied to media, seasonal, macroeconomic variables)', 1, 7, (1, 3), step=1)
+
+# with columns[2]:
+#    slider_value_power=st.slider('Select Power range (only applied to media )',0,4,(1,2),step=1)
+
+# with columns[1]:
+#    st.number_input('Select the range of half saturation point ',min_value=1,max_value=5)
+#    st.number_input('Select the range of  ')
+
+def lag(X, features, min_lag=0,max_lag=6):
+    
+    for i in features:
+        for lag in range(min_lag, max_lag + 1):
+            X[f'{i}_lag{lag}'] = X[i].shift(periods=lag)
+    return X.fillna(method='bfill')
+
+def adstock_variable(X,variable_name,decay):
+    adstock = [0] * len(X[variable_name])  
+
+    for t in range(len(X[variable_name])):
+        if t == 0:
+            adstock[t] = X[variable_name][t] 
+        else:
+            adstock[t] = X[variable_name][t] + adstock[t-1] * decay
+    return adstock
+
+if 'media_data' not in st.session_state:
+   
+  st.session_state['media_data']=pd.DataFrame()
+
+variables_to_be_transformed=[col for col in media_data.columns if col.lower() not in ['dma','panel'] ]
+# st.write(variables_to_be_transformed)
+with columns[0]:
+  if st.button('Apply Transformations'):
+    for dm in dma_dict.keys():
+      dma_dict[dm].reset_index(drop=True,inplace=True)
+      dma_dict[dm]=lag(dma_dict[dm],variables_to_be_transformed,min_lag=slider_value_lag[0],max_lag=slider_value_lag[1])
+
+    for dm in dma_dict.keys():
+      for i in dma_dict[dm].drop(['DMA','Panel'],axis=1).columns:
+        for j in np.arange(slider_value_adstock[0],slider_value_adstock[1]+0.1,0.1):#adding adstock
+          dma_dict[dm][f'{i}_adst.{np.round(j,2)}']=adstock_variable(dma_dict[dm],i,j)
+          
+    
+    st.write(dma_dict)
+    st.session_state['media_data']=media_data
+
+    with st.spinner('Applying Transformations'):
+      time.sleep(2)
+      st.success("Transformations complete!")
+
+if st.session_state['media_data'].shape[1]>old_shape[1]:
+  with columns[0]:
+    st.write(f'Total no.of variables before transformation: {old_shape[1]}, Total no.of variables after transformation: {st.session_state["media_data"].shape[1]}')
+  #st.write(f'Total no.of variables after transformation: {st.session_state["media_data"].shape[1]}')
+
+
+bucket=['paid_search', 'kwai','indicacao','infleux', 'influencer','FB: Level Achieved - Tier 1 Impressions',
+      ' FB: Level Achieved - Tier 2 Impressions','paid_social_others',
+        ' GA App: Will And Cid Pequena Baixo Risco Clicks',
+      'digital_tactic_others',"programmatic"
+      ]
+   
+with columns[1]:
+  if st.button('Create Combinations of Variables'):
+
+    top_3_correlated_features=[]
+    for col in st.session_state['media_data'].columns[:19]:
+        corr_df=pd.concat([st.session_state['media_data'].filter(regex=col),
+                  revenue],axis=1).corr()['Total Approved Accounts - Revenue'].iloc[:-1]
+        top_3_correlated_features.append(list(corr_df.sort_values(ascending=False).head(2).index))
+    flattened_list = [item for sublist in top_3_correlated_features for item in sublist]
+    all_features_set={var:[col for col in flattened_list if var in col] for var in bucket}
+    channels_all=[values for values in all_features_set.values()]
+    st.session_state['combinations'] = list(itertools.product(*channels_all))
+
+  # if 'combinations' not in st.session_state:  
+  #   st.session_state['combinations']=combinations_all
+
+    st.session_state['final_selection']=st.session_state['combinations']
+
+
+
+revenue.reset_index(drop=True,inplace=True)
+if 'Model_results' not in st.session_state:
+      st.session_state['Model_results']={'Model_object':[],
+    'Model_iteration':[],
+    'Feature_set':[],
+    'MAPE':[],
+    'R2':[],
+    'ADJR2':[]
+    }
+
+#if st.button('Build Model'):
+if 'iterations' not in st.session_state:
+   st.session_state['iterations']=1
+save_path = r"Model"
+with columns[1]:
+  if "final_selection" in st.session_state:
+    st.write(f'Total combinations created {format_numbers(len(st.session_state["final_selection"]))}')
+    
+    st.success('Done')
+if st.checkbox('Build all iterations'):
+   iterations=len(st.session_state['final_selection'])
+else:
+   iterations = st.number_input('Select the number of iterations to perform', min_value=1, step=100, value=st.session_state['iterations'])  
+
+st.session_state['iterations']=iterations
+
+
+st.session_state['media_data']=st.session_state['media_data'].fillna(method='ffill')
+if st.button("Build Models"):  
+  st.markdown('Data Split -- Training Period: May 9th, 2023 - October 5th,2023 , Testing Period: October 6th, 2023 - November 7th, 2023 ')
+  progress_bar = st.progress(0)  # Initialize the progress bar
+  #time_remaining_text = st.empty()  # Create an empty space for time remaining text
+  start_time = time.time()  # Record the start time
+  progress_text = st.empty()
+  #time_elapsed_text = st.empty()
+  for i, selected_features in enumerate(st.session_state["final_selection"][40000:40000+int(iterations)]):
+      df = st.session_state['media_data']
+
+      fet = [var for var in selected_features if len(var) > 0]
+      X = df[fet]
+      y = revenue
+      ss = MinMaxScaler()
+      X = pd.DataFrame(ss.fit_transform(X), columns=X.columns)
+      X = sm.add_constant(X)
+      X_train=X.iloc[:150]
+      X_test=X.iloc[150:]
+      y_train=y.iloc[:150]
+      y_test=y.iloc[150:]
+
+
+      model = sm.OLS(y_train, X_train).fit()
+      # st.write(fet)
+      positive_coeff=X.columns
+      negetive_coeff=[]
+      coefficients=model.params.to_dict()
+      model_possitive=[col for col in coefficients.keys() if coefficients[col]>0]
+      # st.write(positive_coeff)
+      # st.write(model_possitive)
+      pvalues=[var for var in list(model.pvalues) if var<=0.06]
+      if (len(model_possitive)/len(selected_features))>0.9 and (len(pvalues)/len(selected_features))>=0.8:
+
+
+          predicted_values = model.predict(X_train)
+          mape = mean_absolute_percentage_error(y_train, predicted_values)
+          adjr2 = model.rsquared_adj
+          r2 = model.rsquared
+          filename = os.path.join(save_path, f"model_{i}.pkl")
+          with open(filename, "wb") as f:
+            pickle.dump(model, f)
+          # with open(r"C:\Users\ManojP\Documents\MMM\simopt\Model\model.pkl", 'rb') as file:
+          #   model = pickle.load(file)
+
+          st.session_state['Model_results']['Model_object'].append(filename)
+          st.session_state['Model_results']['Model_iteration'].append(i)
+          st.session_state['Model_results']['Feature_set'].append(fet)
+          st.session_state['Model_results']['MAPE'].append(mape)
+          st.session_state['Model_results']['R2'].append(r2)
+          st.session_state['Model_results']['ADJR2'].append(adjr2)
+
+      current_time = time.time()
+      time_taken = current_time - start_time
+      time_elapsed_minutes = time_taken / 60
+      completed_iterations_text = f"{i + 1}/{iterations}"
+      progress_bar.progress((i + 1) / int(iterations))
+      progress_text.text(f'Completed iterations: {completed_iterations_text},Time Elapsed (min): {time_elapsed_minutes:.2f}')
+  
+st.write(f'Out of {st.session_state["iterations"]} iterations : {len(st.session_state["Model_results"]["Model_object"])} valid models')
+pd.DataFrame(st.session_state['Model_results']).to_csv('model_output.csv')
+
+def to_percentage(value):
+  return f'{value * 100:.1f}%'   
+
+st.title('2. Select Models')
+if 'tick' not in st.session_state:
+   st.session_state['tick']=False
+if st.checkbox('Show results of top 10 models (based on MAPE and Adj. R2)',value=st.session_state['tick']):
+  st.session_state['tick']=True
+  st.write('Select one model iteration to generate performance metrics for it:')
+  data=pd.DataFrame(st.session_state['Model_results'])
+  data.sort_values(by=['MAPE'],ascending=False,inplace=True)
+  data.drop_duplicates(subset='Model_iteration',inplace=True)
+  top_10=data.head(10)
+  top_10['Rank']=np.arange(1,len(top_10)+1,1)
+  top_10[['MAPE','R2','ADJR2']]=np.round(top_10[['MAPE','R2','ADJR2']],4).applymap(to_percentage)
+  top_10_table = top_10[['Rank','Model_iteration','MAPE','ADJR2','R2']]
+  #top_10_table.columns=[['Rank','Model Iteration Index','MAPE','Adjusted R2','R2']]
+
+  
+  gd=GridOptionsBuilder.from_dataframe(top_10_table)
+  gd.configure_pagination(enabled=True)
+  gd.configure_selection(use_checkbox=True)
+
+  
+  gridoptions=gd.build()
+
+  table = AgGrid(top_10,gridOptions=gridoptions,update_mode=GridUpdateMode.SELECTION_CHANGED)
+  
+  selected_rows=table.selected_rows
+  # if st.session_state["selected_rows"] != selected_rows:
+  #   st.session_state["build_rc_cb"] = False
+  st.session_state["selected_rows"] = selected_rows
+  if 'Model' not in st.session_state:
+    st.session_state['Model']={}
+
+  if len(selected_rows)>0:
+    st.header('2.1 Results Summary')
+
+    model_object=data[data['Model_iteration']==selected_rows[0]['Model_iteration']]['Model_object']
+    features_set=data[data['Model_iteration']==selected_rows[0]['Model_iteration']]['Feature_set']
+    
+    with open(str(model_object.values[0]), 'rb') as file:
+            model = pickle.load(file)
+    st.write(model.summary())        
+    st.header('2.2 Actual vs. Predicted Plot')
+  
+    df=st.session_state['media_data']
+    X=df[features_set.values[0]]
+    X = sm.add_constant(X)
+    y=revenue
+    X_train=X.iloc[:150]
+    X_test=X.iloc[150:]
+    y_train=y.iloc[:150]
+    y_test=y.iloc[150:]
+    ss = MinMaxScaler()
+    X_train = pd.DataFrame(ss.fit_transform(X_train), columns=X_train.columns)
+    st.session_state['X']=X_train
+    st.session_state['features_set']=features_set.values[0]
+
+    metrics_table,line,actual_vs_predicted_plot=plot_actual_vs_predicted(date, y_train, model.predict(X_train), model,target_column='Revenue')
+
+    st.plotly_chart(actual_vs_predicted_plot,use_container_width=True)
+
+
+     
+    st.markdown('## 2.3 Residual Analysis')
+    columns=st.columns(2)
+    with columns[0]:
+      fig=plot_residual_predicted(y_train,model.predict(X_train),X_train)
+      st.plotly_chart(fig)
+    
+    with columns[1]:
+      st.empty()
+      fig = qqplot(y_train,model.predict(X_train))
+      st.plotly_chart(fig)
+
+    with columns[0]:
+      fig=residual_distribution(y_train,model.predict(X_train))
+      st.pyplot(fig)
+    
+
+
+    vif_data = pd.DataFrame()
+    # X=X.drop('const',axis=1)
+    vif_data["Variable"] = X_train.columns
+    vif_data["VIF"] = [variance_inflation_factor(X_train.values, i) for i in range(X_train.shape[1])]
+    vif_data.sort_values(by=['VIF'],ascending=False,inplace=True)
+    vif_data=np.round(vif_data)
+    vif_data['VIF']=vif_data['VIF'].astype(float)
+    st.header('2.4 Variance Inflation Factor (VIF)')
+    #st.dataframe(vif_data)
+    color_mapping = {
+    'darkgreen': (vif_data['VIF'] < 3),
+    'orange': (vif_data['VIF'] >= 3) & (vif_data['VIF'] <= 10),
+    'darkred': (vif_data['VIF'] > 10)
+    }
+
+# Create a horizontal bar plot
+    fig, ax = plt.subplots()
+    fig.set_figwidth(10)  # Adjust the width of the figure as needed
+
+    # Sort the bars by descending VIF values
+    vif_data = vif_data.sort_values(by='VIF', ascending=False)
+
+    # Iterate through the color mapping and plot bars with corresponding colors
+    for color, condition in color_mapping.items():
+        subset = vif_data[condition]
+        bars = ax.barh(subset["Variable"], subset["VIF"], color=color, label=color)
+        
+        # Add text annotations on top of the bars
+        for bar in bars:
+            width = bar.get_width()
+            ax.annotate(f'{width:}', xy=(width, bar.get_y() + bar.get_height() / 2), xytext=(5, 0),
+                        textcoords='offset points', va='center')
+
+    # Customize the plot
+    ax.set_xlabel('VIF Values')
+    #ax.set_title('2.4 Variance Inflation Factor (VIF)')
+    #ax.legend(loc='upper right')
+
+    # Display the plot in Streamlit
+    st.pyplot(fig)
+
+    with st.expander('Results Summary Test data'):
+      ss = MinMaxScaler()
+      X_test = pd.DataFrame(ss.fit_transform(X_test), columns=X_test.columns)
+      st.header('2.2 Actual vs. Predicted Plot')
+
+      metrics_table,line,actual_vs_predicted_plot=plot_actual_vs_predicted(date, y_test, model.predict(X_test), model,target_column='Revenue')
+
+      st.plotly_chart(actual_vs_predicted_plot,use_container_width=True)
+
+      st.markdown('## 2.3 Residual Analysis')
+      columns=st.columns(2)
+      with columns[0]:
+        fig=plot_residual_predicted(revenue,model.predict(X_test),X_test)
+        st.plotly_chart(fig)
+      
+      with columns[1]:
+        st.empty()
+        fig = qqplot(revenue,model.predict(X_test))
+        st.plotly_chart(fig)
+
+      with columns[0]:
+        fig=residual_distribution(revenue,model.predict(X_test))
+        st.pyplot(fig)
+        
+    value=False
+    if st.checkbox('Save this model to tune',key='build_rc_cb'):
+      mod_name=st.text_input('Enter model name')
+      if len(mod_name)>0:
+        st.session_state['Model'][mod_name]={"Model_object":model,'feature_set':st.session_state['features_set'],'X_train':X_train}
+        st.session_state['X_train']=X_train
+        st.session_state['X_test']=X_test
+        st.session_state['y_train']=y_train
+        st.session_state['y_test']=y_test
+        with open("best_models.pkl", "wb") as f:
+          pickle.dump(st.session_state['Model'], f)  
+          st.success('Model saved!, Proceed  next page to tune the model')
+        value=False

dump/3_Model_Tuning.py

@@ -0,0 +1,197 @@
+import streamlit as st
+import pandas as pd
+from Eda_functions import format_numbers
+import pickle
+from utilities import set_header,load_local_css
+import statsmodels.api as sm
+import re
+from sklearn.preprocessing import MinMaxScaler
+import matplotlib.pyplot as plt
+from statsmodels.stats.outliers_influence import variance_inflation_factor
+st.set_option('deprecation.showPyplotGlobalUse', False)
+from Data_prep_functions import *
+
+st.set_page_config(
+  page_title="Model Tuning",
+  page_icon=":shark:",
+  layout="wide",
+  initial_sidebar_state='collapsed'
+)
+load_local_css('styles.css')
+set_header()
+
+
+st.title('1. Model Tuning')
+
+
+if "X_train" not in st.session_state:
+   st.error(
+"Oops! It seems there are no saved models available. Please build and save a model from the previous page to proceed.")
+   st.stop()
+X_train=st.session_state['X_train']
+X_test=st.session_state['X_test']
+y_train=st.session_state['y_train']
+y_test=st.session_state['y_test']
+df=st.session_state['media_data']
+
+with open("best_models.pkl", 'rb') as file:
+  model_dict= pickle.load(file)
+
+if 'selected_model' not in st.session_state:
+   st.session_state['selected_model']=0
+
+
+st.markdown('### 1.1 Event Flags')
+st.markdown('Helps in quantifying the impact of specific occurrences of events')   
+with st.expander('Apply Event Flags'):
+  st.session_state["selected_model"]=st.selectbox('Select Model to apply flags',model_dict.keys())
+  model =model_dict[st.session_state["selected_model"]]['Model_object']
+  date=st.session_state['date']
+  date=pd.to_datetime(date)
+  X_train =model_dict[st.session_state["selected_model"]]['X_train']
+  features_set= model_dict[st.session_state["selected_model"]]['feature_set']
+
+  col=st.columns(3)  
+  min_date=min(date)
+  max_date=max(date)
+  with col[0]:
+    start_date=st.date_input('Select Start Date',min_date,min_value=min_date,max_value=max_date)
+  with col[1]:
+    end_date=st.date_input('Select End Date',max_date,min_value=min_date,max_value=max_date)
+  with col[2]:
+    repeat=st.selectbox('Repeat Annually',['Yes','No'],index=1)
+  if repeat =='Yes':
+      repeat=True
+  else: 
+      repeat=False
+  # X_train=sm.add_constant(X_train)
+
+  if 'Flags' not in st.session_state:
+    st.session_state['Flags']={}
+
+  met,line_values,fig_flag=plot_actual_vs_predicted(date[:150], y_train, model.predict(X_train), model,flag=(start_date,end_date),repeat_all_years=repeat)
+  st.plotly_chart(fig_flag,use_container_width=True)
+  flag_name='f1'
+  flag_name=st.text_input('Enter Flag Name')
+  if st.button('Update flag'):
+    st.session_state['Flags'][flag_name]=line_values
+    st.success(f'{flag_name} stored')
+
+  options=list(st.session_state['Flags'].keys())
+  selected_options = []
+  num_columns = 4
+  num_rows = -(-len(options) // num_columns)  
+
+
+tick=False
+if st.checkbox('Select all'):
+    tick=True
+selected_options = []
+for row in range(num_rows):
+    cols = st.columns(num_columns)
+    for col in cols:
+        if options:
+            option = options.pop(0) 
+            selected = col.checkbox(option,value=tick)
+            if selected:
+                selected_options.append(option)
+
+st.markdown('### 1.2 Select Parameters to Apply')
+parameters=st.columns(3)
+with parameters[0]:
+   Trend=st.checkbox("**Trend**")
+   st.markdown('Helps account for long-term trends or seasonality that could influence advertising effectiveness')
+with parameters[1]:
+   week_number=st.checkbox('**Week_number**')
+   st.markdown('Assists in detecting and incorporating weekly patterns or seasonality')
+with parameters[2]:
+   sine_cosine=st.checkbox('**Sine and Cosine Waves**')
+   st.markdown('Helps in capturing cyclical patterns or seasonality in the data')
+if st.button('Build model with Selected Parameters and Flags'):
+  st.header('2.1 Results Summary')
+  # date=list(df.index)
+  # df = df.reset_index(drop=True)
+  # st.write(df.head(2))
+  # X_train=df[features_set]
+  ss = MinMaxScaler()
+  X_train_tuned = pd.DataFrame(ss.fit_transform(X_train), columns=X_train.columns)
+  X_train_tuned=sm.add_constant(X_train_tuned)
+  for flag in selected_options:
+    X_train_tuned[flag]=st.session_state['Flags'][flag]
+  if Trend:
+     X_train_tuned['Trend']=np.arange(1,len(X_train_tuned)+1,1)
+  # if week_number:
+  #    st.write(date)
+     date=pd.to_datetime(date.values)
+     X_train_tuned['Week_number']=date.day_of_week[:150]
+  model_tuned = sm.OLS(y_train, X_train_tuned).fit()
+
+  metrics_table,line,actual_vs_predicted_plot=plot_actual_vs_predicted(date[:150], y_train, model.predict(X_train), model,target_column='Revenue')
+  metrics_table_tuned,line,actual_vs_predicted_plot_tuned=plot_actual_vs_predicted(date[:150], y_train, model_tuned.predict(X_train_tuned), model_tuned,target_column='Revenue')
+  
+  # st.write(metrics_table)
+  mape=np.round(metrics_table.iloc[0,1],2)
+  r2=np.round(metrics_table.iloc[1,1],2)
+  adjr2=np.round(metrics_table.iloc[2,1],2)
+  mape_tuned=np.round(metrics_table_tuned.iloc[0,1],2)
+  r2_tuned=np.round(metrics_table_tuned.iloc[1,1],2)
+  adjr2_tuned=np.round(metrics_table_tuned.iloc[2,1],2)
+  parameters_=st.columns(3)
+  with parameters_[0]:
+     st.metric('R2',r2_tuned,np.round(r2_tuned-r2,2))
+  with parameters_[1]:
+     st.metric('Adjusted R2',adjr2_tuned,np.round(adjr2_tuned-adjr2,2))
+  with parameters_[2]:
+     st.metric('MAPE',mape_tuned,np.round(mape_tuned-mape,2),'inverse')
+
+  st.header('2.2 Actual vs. Predicted Plot')
+  metrics_table,line,actual_vs_predicted_plot=plot_actual_vs_predicted(date, y_train, model.predict(X_train), model,target_column='Revenue')
+
+  st.plotly_chart(actual_vs_predicted_plot,use_container_width=True)
+
+
+     
+  st.markdown('## 2.3 Residual Analysis')
+  columns=st.columns(2)
+  with columns[0]:
+    fig=plot_residual_predicted(y_train,model.predict(X_train),X_train)
+    st.plotly_chart(fig)
+  
+  with columns[1]:
+    st.empty()
+    fig = qqplot(y_train,model.predict(X_train))
+    st.plotly_chart(fig)
+
+  with columns[0]:
+    fig=residual_distribution(y_train,model.predict(X_train))
+    st.pyplot(fig)
+
+# if st.checkbox('Use this model to build response curves',key='123'):
+
+#   raw_data=df[features_set]
+#   columns_raw=[re.split(r"(_lag|_adst)",col)[0] for col in raw_data.columns]
+#   raw_data.columns=columns_raw
+#   columns_media=[col for col in columns_raw if Categorised_data[col]['BB']=='Media']
+#   raw_data=raw_data[columns_media]
+
+#   raw_data['Date']=list(df.index)
+
+#   spends_var=[col for col in df.columns if "spends" in col.lower() and 'adst' not in col.lower() and 'lag' not in col.lower()]
+#   spends_df=df[spends_var]
+#   spends_df['Week']=list(df.index)
+  
+ 
+#   j=0
+#   X1=X.copy()
+#   col=X1.columns
+#   for i in model.params.values:
+#       X1[col[j]]=X1.iloc[:,j]*i
+#       j+=1
+#   contribution_df=X1
+#   contribution_df['Date']=list(df.index)
+#   excel_file='Overview_data.xlsx'
+
+#   with pd.ExcelWriter(excel_file,engine='xlsxwriter') as writer:
+#      raw_data.to_excel(writer,sheet_name='RAW DATA MMM',index=False)
+#      spends_df.to_excel(writer,sheet_name='SPEND INPUT',index=False)
+#      contribution_df.to_excel(writer,sheet_name='CONTRIBUTION MMM')