Rename pages/Data_Import.py to pages/Help.py

pages/Data_Import.py

@@ -1,891 +0,0 @@
-# Importing necessary libraries
-import streamlit as st
-
-st.set_page_config(
-    page_title="Model Build",
-    page_icon=":shark:",
-    layout="wide",
-    initial_sidebar_state="collapsed",
-)
-
-import numpy as np
-import pandas as pd
-from utilities import set_header, load_local_css, load_authenticator
-import pickle
-
-
-load_local_css("styles.css")
-set_header()
-
-authenticator = st.session_state.get("authenticator")
-if authenticator is None:
-    authenticator = load_authenticator()
-
-name, authentication_status, username = authenticator.login("Login", "main")
-auth_status = st.session_state.get("authentication_status")
-
-# Check for authentication status
-if auth_status != True:
-    st.stop()
-
-
-# Function to validate date column in dataframe
-def validate_date_column(df):
-    try:
-        # Attempt to convert the 'Date' column to datetime
-        df["date"] = pd.to_datetime(df["date"], format="%d-%m-%Y")
-        return True
-    except:
-        return False
-
-
-# Function to determine data interval
-def determine_data_interval(common_freq):
-    if common_freq == 1:
-        return "daily"
-    elif common_freq == 7:
-        return "weekly"
-    elif 28 <= common_freq <= 31:
-        return "monthly"
-    else:
-        return "irregular"
-
-
-# Function to read each uploaded Excel file into a pandas DataFrame and stores them in a dictionary
-st.cache_resource(show_spinner=False)
-
-
-def files_to_dataframes(uploaded_files):
-    df_dict = {}
-    for uploaded_file in uploaded_files:
-        # Extract file name without extension
-        file_name = uploaded_file.name.rsplit(".", 1)[0]
-
-        # Check for duplicate file names
-        if file_name in df_dict:
-            st.warning(
-                f"Duplicate File: {file_name}. This file will be skipped.",
-                icon="⚠️",
-            )
-            continue
-
-        # Read the file into a DataFrame
-        df = pd.read_excel(uploaded_file)
-
-        # Convert all column names to lowercase
-        df.columns = df.columns.str.lower().str.strip()
-
-        # Separate numeric and non-numeric columns
-        numeric_cols = list(df.select_dtypes(include=["number"]).columns)
-        non_numeric_cols = [
-            col
-            for col in df.select_dtypes(exclude=["number"]).columns
-            if col.lower() != "date"
-        ]
-
-        # Check for 'Date' column
-        if not (validate_date_column(df) and len(numeric_cols) > 0):
-            st.warning(
-                f"File Name: {file_name} ➜ Please upload data with Date column in 'DD-MM-YYYY' format and at least one media/exogenous column. This file will be skipped.",
-                icon="⚠️",
-            )
-            continue
-
-        # Check for interval
-        common_freq = common_freq = (
-            pd.Series(df["date"].unique()).diff().dt.days.dropna().mode()[0]
-        )
-        # Calculate the data interval (daily, weekly, monthly or irregular)
-        interval = determine_data_interval(common_freq)
-        if interval == "irregular":
-            st.warning(
-                f"File Name: {file_name} ➜ Please upload data in daily, weekly or monthly interval. This file will be skipped.",
-                icon="⚠️",
-            )
-            continue
-
-        # Store both DataFrames in the dictionary under their respective keys
-        df_dict[file_name] = {
-            "numeric": numeric_cols,
-            "non_numeric": non_numeric_cols,
-            "interval": interval,
-            "df": df,
-        }
-
-    return df_dict
-
-
-# Function to adjust dataframe granularity
-# def adjust_dataframe_granularity(df, current_granularity, target_granularity):
-#     # Set index
-#     df.set_index("date", inplace=True)
-
-#     # Define aggregation rules for resampling
-#     aggregation_rules = {
-#         col: "sum" if pd.api.types.is_numeric_dtype(df[col]) else "first"
-#         for col in df.columns
-#     }
-
-#     resampled_df = df
-#     if current_granularity == "daily" and target_granularity == "weekly":
-#         resampled_df = df.resample("W-MON").agg(aggregation_rules)
-
-#     elif current_granularity == "daily" and target_granularity == "monthly":
-#         resampled_df = df.resample("MS").agg(aggregation_rules)
-
-#     elif current_granularity == "daily" and target_granularity == "daily":
-#         resampled_df = df.resample("D").agg(aggregation_rules)
-
-#     elif current_granularity in ["weekly", "monthly"] and target_granularity == "daily":
-#         # For higher to lower granularity, distribute numeric and replicate non-numeric values equally across the new period
-#         expanded_data = []
-#         for _, row in df.iterrows():
-#             if current_granularity == "weekly":
-#                 period_range = pd.date_range(start=row.name, periods=7)
-#             elif current_granularity == "monthly":
-#                 period_range = pd.date_range(
-#                     start=row.name, periods=row.name.days_in_month
-#                 )
-
-#             for date in period_range:
-#                 new_row = {}
-#                 for col in df.columns:
-#                     if pd.api.types.is_numeric_dtype(df[col]):
-#                         if current_granularity == "weekly":
-#                             new_row[col] = row[col] / 7
-#                         elif current_granularity == "monthly":
-#                             new_row[col] = row[col] / row.name.days_in_month
-#                     else:
-#                         new_row[col] = row[col]
-#                 expanded_data.append((date, new_row))
-
-#         resampled_df = pd.DataFrame(
-#             [data for _, data in expanded_data],
-#             index=[date for date, _ in expanded_data],
-#         )
-
-#     # Reset index
-#     resampled_df = resampled_df.reset_index().rename(columns={"index": "date"})
-
-#     return resampled_df
-
-
-def adjust_dataframe_granularity(df, current_granularity, target_granularity):
-    # Set index
-    df.set_index("date", inplace=True)
-
-    # Define aggregation rules for resampling
-    aggregation_rules = {
-        col: "sum" if pd.api.types.is_numeric_dtype(df[col]) else "first"
-        for col in df.columns
-    }
-
-    # Initialize resampled_df
-    resampled_df = df
-    if current_granularity == "daily" and target_granularity == "weekly":
-        resampled_df = df.resample("W-MON", closed="left", label="left").agg(
-            aggregation_rules
-        )
-
-    elif current_granularity == "daily" and target_granularity == "monthly":
-        resampled_df = df.resample("MS", closed="left", label="left").agg(
-            aggregation_rules
-        )
-
-    elif current_granularity == "daily" and target_granularity == "daily":
-        resampled_df = df.resample("D").agg(aggregation_rules)
-
-    elif current_granularity in ["weekly", "monthly"] and target_granularity == "daily":
-        # For higher to lower granularity, distribute numeric and replicate non-numeric values equally across the new period
-        expanded_data = []
-        for _, row in df.iterrows():
-            if current_granularity == "weekly":
-                period_range = pd.date_range(start=row.name, periods=7)
-            elif current_granularity == "monthly":
-                period_range = pd.date_range(
-                    start=row.name, periods=row.name.days_in_month
-                )
-
-            for date in period_range:
-                new_row = {}
-                for col in df.columns:
-                    if pd.api.types.is_numeric_dtype(df[col]):
-                        if current_granularity == "weekly":
-                            new_row[col] = row[col] / 7
-                        elif current_granularity == "monthly":
-                            new_row[col] = row[col] / row.name.days_in_month
-                    else:
-                        new_row[col] = row[col]
-                expanded_data.append((date, new_row))
-
-        resampled_df = pd.DataFrame(
-            [data for _, data in expanded_data],
-            index=[date for date, _ in expanded_data],
-        )
-
-    # Reset index
-    resampled_df = resampled_df.reset_index().rename(columns={"index": "date"})
-
-    return resampled_df
-
-
-# Function to clean and extract unique values of DMA and Panel
-st.cache_resource(show_spinner=False)
-
-
-def clean_and_extract_unique_values(files_dict, selections):
-    all_dma_values = set()
-    all_panel_values = set()
-
-    for file_name, file_data in files_dict.items():
-        df = file_data["df"]
-
-        # 'DMA' and 'Panel' selections
-        selected_dma = selections[file_name].get("DMA")
-        selected_panel = selections[file_name].get("Panel")
-
-        # Clean and standardize DMA column if it exists and is selected
-        if selected_dma and selected_dma != "N/A" and selected_dma in df.columns:
-            df[selected_dma] = (
-                df[selected_dma].str.lower().str.strip().str.replace("_", " ")
-            )
-            all_dma_values.update(df[selected_dma].dropna().unique())
-
-        # Clean and standardize Panel column if it exists and is selected
-        if selected_panel and selected_panel != "N/A" and selected_panel in df.columns:
-            df[selected_panel] = (
-                df[selected_panel].str.lower().str.strip().str.replace("_", " ")
-            )
-            all_panel_values.update(df[selected_panel].dropna().unique())
-
-        # Update the processed DataFrame back in the dictionary
-        files_dict[file_name]["df"] = df
-
-    return all_dma_values, all_panel_values
-
-
-# Function to format values for display
-st.cache_resource(show_spinner=False)
-
-
-def format_values_for_display(values_list):
-    # Capitalize the first letter of each word and replace underscores with spaces
-    formatted_list = [value.replace("_", " ").title() for value in values_list]
-    # Join values with commas and 'and' before the last value
-    if len(formatted_list) > 1:
-        return ", ".join(formatted_list[:-1]) + ", and " + formatted_list[-1]
-    elif formatted_list:
-        return formatted_list[0]
-    return "No values available"
-
-
-# Function to normalizes all data within files_dict to a daily granularity
-st.cache(show_spinner=False, allow_output_mutation=True)
-
-
-def standardize_data_to_daily(files_dict, selections):
-    # Normalize all data to a daily granularity using a provided function
-    files_dict = apply_granularity_to_all(files_dict, "daily", selections)
-
-    # Update the "interval" attribute for each dataset to indicate the new granularity
-    for files_name, files_data in files_dict.items():
-        files_data["interval"] = "daily"
-
-    return files_dict
-
-
-# Function to apply granularity transformation to all DataFrames in files_dict
-st.cache_resource(show_spinner=False)
-
-
-def apply_granularity_to_all(files_dict, granularity_selection, selections):
-    for file_name, file_data in files_dict.items():
-        df = file_data["df"].copy()
-
-        # Handling when DMA or Panel might be 'N/A'
-        selected_dma = selections[file_name].get("DMA")
-        selected_panel = selections[file_name].get("Panel")
-
-        # Correcting the segment selection logic & handling 'N/A'
-        if selected_dma != "N/A" and selected_panel != "N/A":
-            unique_combinations = df[[selected_dma, selected_panel]].drop_duplicates()
-        elif selected_dma != "N/A":
-            unique_combinations = df[[selected_dma]].drop_duplicates()
-            selected_panel = None  # Ensure Panel is ignored if N/A
-        elif selected_panel != "N/A":
-            unique_combinations = df[[selected_panel]].drop_duplicates()
-            selected_dma = None  # Ensure DMA is ignored if N/A
-        else:
-            # If both are 'N/A', process the entire dataframe as is
-            df = adjust_dataframe_granularity(
-                df, file_data["interval"], granularity_selection
-            )
-            files_dict[file_name]["df"] = df
-            continue  # Skip to the next file
-
-        transformed_segments = []
-        for _, combo in unique_combinations.iterrows():
-            if selected_dma and selected_panel:
-                segment = df[
-                    (df[selected_dma] == combo[selected_dma])
-                    & (df[selected_panel] == combo[selected_panel])
-                ]
-            elif selected_dma:
-                segment = df[df[selected_dma] == combo[selected_dma]]
-            elif selected_panel:
-                segment = df[df[selected_panel] == combo[selected_panel]]
-
-            # Adjust granularity of the segment
-            transformed_segment = adjust_dataframe_granularity(
-                segment, file_data["interval"], granularity_selection
-            )
-            transformed_segments.append(transformed_segment)
-
-        # Combine all transformed segments into a single DataFrame for this file
-        transformed_df = pd.concat(transformed_segments, ignore_index=True)
-        files_dict[file_name]["df"] = transformed_df
-
-    return files_dict
-
-
-# Function to create main dataframe structure
-st.cache_resource(show_spinner=False)
-
-
-def create_main_dataframe(
-    files_dict, all_dma_values, all_panel_values, granularity_selection
-):
-    # Determine the global start and end dates across all DataFrames
-    global_start = min(df["df"]["date"].min() for df in files_dict.values())
-    global_end = max(df["df"]["date"].max() for df in files_dict.values())
-
-    # Adjust the date_range generation based on the granularity_selection
-    if granularity_selection == "weekly":
-        # Generate a weekly range, with weeks starting on Monday
-        date_range = pd.date_range(start=global_start, end=global_end, freq="W-MON")
-    elif granularity_selection == "monthly":
-        # Generate a monthly range, starting from the first day of each month
-        date_range = pd.date_range(start=global_start, end=global_end, freq="MS")
-    else:  # Default to daily if not weekly or monthly
-        date_range = pd.date_range(start=global_start, end=global_end, freq="D")
-
-    # Collect all unique DMA and Panel values, excluding 'N/A'
-    all_dmas = all_dma_values
-    all_panels = all_panel_values
-
-    # Dynamically build the list of dimensions (Panel, DMA) to include in the main DataFrame based on availability
-    dimensions, merge_keys = [], []
-    if all_panels:
-        dimensions.append(all_panels)
-        merge_keys.append("Panel")
-    if all_dmas:
-        dimensions.append(all_dmas)
-        merge_keys.append("DMA")
-
-    dimensions.append(date_range)  # Date range is always included
-    merge_keys.append("date")  # Date range is always included
-
-    # Create a main DataFrame template with the dimensions
-    main_df = pd.MultiIndex.from_product(
-        dimensions,
-        names=[name for name, _ in zip(merge_keys, dimensions)],
-    ).to_frame(index=False)
-
-    return main_df.reset_index(drop=True)
-
-
-# Function to prepare and merge dataFrames
-st.cache_resource(show_spinner=False)
-
-
-def merge_into_main_df(main_df, files_dict, selections):
-    for file_name, file_data in files_dict.items():
-        df = file_data["df"].copy()
-
-        # Rename selected DMA and Panel columns if not 'N/A'
-        selected_dma = selections[file_name].get("DMA", "N/A")
-        selected_panel = selections[file_name].get("Panel", "N/A")
-        if selected_dma != "N/A":
-            df.rename(columns={selected_dma: "DMA"}, inplace=True)
-        if selected_panel != "N/A":
-            df.rename(columns={selected_panel: "Panel"}, inplace=True)
-
-        # Merge current DataFrame into main_df based on 'date', and where applicable, 'Panel' and 'DMA'
-        merge_keys = ["date"]
-        if "Panel" in df.columns:
-            merge_keys.append("Panel")
-        if "DMA" in df.columns:
-            merge_keys.append("DMA")
-        main_df = pd.merge(main_df, df, on=merge_keys, how="left")
-
-    # After all merges, sort by 'date' and reset index for cleanliness
-    sort_by = ["date"]
-    if "Panel" in main_df.columns:
-        sort_by.append("Panel")
-    if "DMA" in main_df.columns:
-        sort_by.append("DMA")
-    main_df.sort_values(by=sort_by, inplace=True)
-    main_df.reset_index(drop=True, inplace=True)
-
-    return main_df
-
-
-# Function to categorize column
-def categorize_column(column_name):
-    # Define keywords for each category
-    internal_keywords = [
-        "Price",
-        "Discount",
-        "product_price",
-        "cost",
-        "margin",
-        "inventory",
-        "sales",
-        "revenue",
-        "turnover",
-        "expense",
-    ]
-    exogenous_keywords = [
-        "GDP",
-        "Tax",
-        "Inflation",
-        "interest_rate",
-        "employment_rate",
-        "exchange_rate",
-        "consumer_spending",
-        "retail_sales",
-        "oil_prices",
-        "weather",
-    ]
-
-    # Check if the column name matches any of the keywords for Internal or Exogenous categories
-    for keyword in internal_keywords:
-        if keyword.lower() in column_name.lower():
-            return "Internal"
-    for keyword in exogenous_keywords:
-        if keyword.lower() in column_name.lower():
-            return "Exogenous"
-
-    # Default to Media if no match found
-    return "Media"
-
-
-# Function to calculate missing stats and prepare for editable DataFrame
-st.cache_resource(show_spinner=False)
-
-
-def prepare_missing_stats_df(df):
-    missing_stats = []
-    for column in df.columns:
-        if (
-            column == "date" or column == "DMA" or column == "Panel"
-        ):  # Skip Date, DMA and Panel column
-            continue
-
-        missing = df[column].isnull().sum()
-        pct_missing = round((missing / len(df)) * 100, 2)
-
-        # Dynamically assign category based on column name
-        # category = categorize_column(column)
-        category = "Media"
-
-        missing_stats.append(
-            {
-                "Column": column,
-                "Missing Values": missing,
-                "Missing Percentage": pct_missing,
-                "Impute Method": "Fill with 0",  # Default value
-                "Category": category,
-            }
-        )
-    stats_df = pd.DataFrame(missing_stats)
-
-    return stats_df
-
-
-# Function to add API DataFrame details to the files dictionary
-st.cache_resource(show_spinner=False)
-
-
-def add_api_dataframe_to_dict(main_df, files_dict):
-    files_dict["API"] = {
-        "numeric": list(main_df.select_dtypes(include=["number"]).columns),
-        "non_numeric": [
-            col
-            for col in main_df.select_dtypes(exclude=["number"]).columns
-            if col.lower() != "date"
-        ],
-        "interval": determine_data_interval(
-            pd.Series(main_df["date"].unique()).diff().dt.days.dropna().mode()[0]
-        ),
-        "df": main_df,
-    }
-
-    return files_dict
-
-
-# Function to reads an API into a DataFrame, parsing specified columns as datetime
-@st.cache_resource(show_spinner=False)
-def read_API_data():
-    return pd.read_excel(r"upf_data_converted.xlsx", parse_dates=["Date"])
-
-
-# Function to set the 'DMA_Panel_Selected' session state variable to False
-def set_DMA_Panel_Selected_false():
-    st.session_state["DMA_Panel_Selected"] = False
-
-
-# Initialize 'final_df' in session state
-if "final_df" not in st.session_state:
-    st.session_state["final_df"] = pd.DataFrame()
-
-# Initialize 'bin_dict' in session state
-if "bin_dict" not in st.session_state:
-    st.session_state["bin_dict"] = {}
-
-# Initialize 'DMA_Panel_Selected' in session state
-if "DMA_Panel_Selected" not in st.session_state:
-    st.session_state["DMA_Panel_Selected"] = False
-
-# Page Title
-st.write("")  # Top padding
-st.title("Data Import")
-
-
-#########################################################################################################################################################
-# Create a dictionary to hold all DataFrames and collect user input to specify "DMA" and "Panel" columns for each file
-#########################################################################################################################################################
-
-
-# Read the Excel file, parsing 'Date' column as datetime
-main_df = read_API_data()
-
-# Convert all column names to lowercase
-main_df.columns = main_df.columns.str.lower().str.strip()
-
-# File uploader
-uploaded_files = st.file_uploader(
-    "Upload additional data",
-    type=["xlsx"],
-    accept_multiple_files=True,
-    on_change=set_DMA_Panel_Selected_false,
-)
-
-# Custom HTML for upload instructions
-recommendation_html = f"""
-<div style="text-align: justify;">
-<strong>Recommendation:</strong> For optimal processing, please ensure that all uploaded datasets including DMA, Panel, media, internal, and exogenous data adhere to the following guidelines: Each dataset must include a <code>Date</code> column formatted as <code>DD-MM-YYYY</code>, be free of missing values.
-</div>
-"""
-st.markdown(recommendation_html, unsafe_allow_html=True)
-
-# Choose Date Granularity
-st.markdown("#### Choose Date Granularity")
-# Granularity Selection
-granularity_selection = st.selectbox(
-    "Choose Date Granularity",
-    ["Daily", "Weekly", "Monthly"],
-    label_visibility="collapsed",
-    on_change=set_DMA_Panel_Selected_false,
-)
-granularity_selection = str(granularity_selection).lower()
-
-# Convert files to dataframes
-files_dict = files_to_dataframes(uploaded_files)
-
-# Add API Dataframe
-if main_df is not None:
-    files_dict = add_api_dataframe_to_dict(main_df, files_dict)
-
-# Display a warning message if no files have been uploaded and halt further execution
-if not files_dict:
-    st.warning(
-        "Please upload at least one file to proceed.",
-        icon="⚠️",
-    )
-    st.stop()  # Halts further execution until file is uploaded
-
-
-# Select DMA and Panel columns
-st.markdown("#### Select DMA and Panel columns")
-selections = {}
-with st.expander("Select DMA and Panel columns", expanded=False):
-    count = 0  # Initialize counter to manage the visibility of labels and keys
-    for file_name, file_data in files_dict.items():
-        # Determine visibility of the label based on the count
-        if count == 0:
-            label_visibility = "visible"
-        else:
-            label_visibility = "collapsed"
-
-        # Extract non-numeric columns
-        non_numeric_cols = file_data["non_numeric"]
-
-        # Prepare DMA and Panel values for dropdown, adding "N/A" as an option
-        dma_values = non_numeric_cols + ["N/A"]
-        panel_values = non_numeric_cols + ["N/A"]
-
-        # Skip if only one option is available
-        if len(dma_values) == 1 and len(panel_values) == 1:
-            selected_dma, selected_panel = "N/A", "N/A"
-            # Update the selections for DMA and Panel for the current file
-            selections[file_name] = {
-                "DMA": selected_dma,
-                "Panel": selected_panel,
-            }
-            continue
-
-        # Create layout columns for File Name, DMA, and Panel selections
-        file_name_col, DMA_col, Panel_col = st.columns([2, 4, 4])
-
-        with file_name_col:
-            # Display "File Name" label only for the first file
-            if count == 0:
-                st.write("File Name")
-            else:
-                st.write("")
-            st.write(file_name)  # Display the file name
-
-        with DMA_col:
-            # Display a selectbox for DMA values
-            selected_dma = st.selectbox(
-                "Select DMA",
-                dma_values,
-                on_change=set_DMA_Panel_Selected_false,
-                label_visibility=label_visibility,  # Control visibility of the label
-                key=f"DMA_selectbox{count}",  # Ensure unique key for each selectbox
-            )
-
-        with Panel_col:
-            # Display a selectbox for Panel values
-            selected_panel = st.selectbox(
-                "Select Panel",
-                panel_values,
-                on_change=set_DMA_Panel_Selected_false,
-                label_visibility=label_visibility,  # Control visibility of the label
-                key=f"Panel_selectbox{count}",  # Ensure unique key for each selectbox
-            )
-
-        # Skip processing if the same column is selected for both Panel and DMA due to potential data integrity issues
-        if selected_panel == selected_dma and not (
-            selected_panel == "N/A" and selected_dma == "N/A"
-        ):
-            st.warning(
-                f"File: {file_name} → The same column cannot serve as both Panel and DMA. Please adjust your selections.",
-            )
-            selected_dma, selected_panel = "N/A", "N/A"
-            st.stop()
-
-        # Update the selections for DMA and Panel for the current file
-        selections[file_name] = {
-            "DMA": selected_dma,
-            "Panel": selected_panel,
-        }
-
-        count += 1  # Increment the counter after processing each file
-
-    # Accept DMA and Panel selection
-    if st.button("Accept and Process", use_container_width=True):
-
-        # Normalize all data to a daily granularity. This initial standardization simplifies subsequent conversions to other levels of granularity
-        with st.spinner("Processing...", cache=True):
-            files_dict = standardize_data_to_daily(files_dict, selections)
-
-            # Convert all data to daily level granularity
-            files_dict = apply_granularity_to_all(
-                files_dict, granularity_selection, selections
-            )
-
-        st.session_state["files_dict"] = files_dict
-        st.session_state["DMA_Panel_Selected"] = True
-
-
-#########################################################################################################################################################
-# Display unique DMA and Panel values
-#########################################################################################################################################################
-
-
-# Halts further execution until DMA and Panel columns are selected
-if "files_dict" in st.session_state and st.session_state["DMA_Panel_Selected"]:
-    files_dict = st.session_state["files_dict"]
-else:
-    st.stop()
-
-# Set to store unique values of DMA and Panel
-with st.spinner("Fetching DMA and Panel values..."):
-    all_dma_values, all_panel_values = clean_and_extract_unique_values(
-        files_dict, selections
-    )
-
-    # List of DMA and Panel columns unique values
-    list_of_all_dma_values = list(all_dma_values)
-    list_of_all_panel_values = list(all_panel_values)
-
-    # Format DMA and Panel values for display
-    formatted_dma_values = format_values_for_display(list_of_all_dma_values)
-    formatted_panel_values = format_values_for_display(list_of_all_panel_values)
-
-# Unique DMA and Panel values
-st.markdown("#### Unique DMA and Panel values")
-# Display DMA and Panel values
-with st.expander("Unique DMA and Panel values"):
-    st.write("")
-    st.markdown(
-        f"""
-    <style>
-    .justify-text {{
-    text-align: justify;
-    }}
-    </style>
-    <div class="justify-text">
-    <strong>Panel Values:</strong> {formatted_panel_values}<br>
-    <strong>DMA Values:</strong> {formatted_dma_values}
-    </div>
-    """,
-        unsafe_allow_html=True,
-    )
-
-    # Display total DMA and Panel
-    st.write("")
-    st.markdown(
-        f"""
-    <div style="text-align: justify;">
-        <strong>Number of DMAs detected:</strong> {len(list_of_all_dma_values)}<br>
-        <strong>Number of Panels detected:</strong> {len(list_of_all_panel_values)}
-    </div>
-    """,
-        unsafe_allow_html=True,
-    )
-    st.write("")
-
-
-#########################################################################################################################################################
-# Merge all DataFrames
-#########################################################################################################################################################
-
-
-# Merge all DataFrames selected
-main_df = create_main_dataframe(
-    files_dict, all_dma_values, all_panel_values, granularity_selection
-)
-merged_df = merge_into_main_df(main_df, files_dict, selections)
-
-# # Display the merged DataFrame
-# st.markdown("#### Merged DataFrame based on selected DMA and Panel")
-# st.dataframe(merged_df)
-
-
-#########################################################################################################################################################
-# Categorize Variables and Impute Missing Values
-#########################################################################################################################################################
-
-
-# Create an editable DataFrame in Streamlit
-st.markdown("#### Select Variables Category & Impute Missing Values")
-
-# Prepare missing stats DataFrame for editing
-missing_stats_df = prepare_missing_stats_df(merged_df)
-
-edited_stats_df = st.data_editor(
-    missing_stats_df,
-    column_config={
-        "Impute Method": st.column_config.SelectboxColumn(
-            options=[
-                "Drop Column",
-                "Fill with Mean",
-                "Fill with Median",
-                "Fill with 0",
-            ],
-            required=True,
-            default="Fill with 0",
-        ),
-        "Category": st.column_config.SelectboxColumn(
-            options=[
-                "Media",
-                "Exogenous",
-                "Internal",
-                "Response_Metric"
-            ],
-            required=True,
-            default="Media",
-        ),
-    },
-    disabled=["Column", "Missing Values", "Missing Percentage"],
-    hide_index=True,
-    use_container_width=True,
-)
-
-# Apply changes based on edited DataFrame
-for i, row in edited_stats_df.iterrows():
-    column = row["Column"]
-    if row["Impute Method"] == "Drop Column":
-        merged_df.drop(columns=[column], inplace=True)
-
-    elif row["Impute Method"] == "Fill with Mean":
-        merged_df[column].fillna(merged_df[column].mean(), inplace=True)
-
-    elif row["Impute Method"] == "Fill with Median":
-        merged_df[column].fillna(merged_df[column].median(), inplace=True)
-
-    elif row["Impute Method"] == "Fill with 0":
-        merged_df[column].fillna(0, inplace=True)
-
-# Display the Final DataFrame and exogenous variables
-st.markdown("#### Final DataFrame")
-final_df = merged_df
-st.dataframe(final_df, hide_index=True)
-
-# Initialize an empty dictionary to hold categories and their variables
-category_dict = {}
-
-# Iterate over each row in the edited DataFrame to populate the dictionary
-for i, row in edited_stats_df.iterrows():
-    column = row["Column"]
-    category = row["Category"]  # The category chosen by the user for this variable
-
-    # Check if the category already exists in the dictionary
-    if category not in category_dict:
-        # If not, initialize it with the current column as its first element
-        category_dict[category] = [column]
-    else:
-        # If it exists, append the current column to the list of variables under this category
-        category_dict[category].append(column)
-
-# Add Date, DMA and Panel in category dictionary
-category_dict.update({"Date": ["date"]})
-if "DMA" in final_df.columns:
-    category_dict["DMA"] = ["DMA"]
-
-if "Panel" in final_df.columns:
-    category_dict["Panel"] = ["Panel"]
-
-# Display the dictionary
-st.markdown("#### Variable Category")
-for category, variables in category_dict.items():
-    # Check if there are multiple variables to handle "and" insertion correctly
-    if len(variables) > 1:
-        # Join all but the last variable with ", ", then add " and " before the last variable
-        variables_str = ", ".join(variables[:-1]) + " and " + variables[-1]
-    else:
-        # If there's only one variable, no need for "and"
-        variables_str = variables[0]
-
-    # Display the category and its variables in the desired format
-    st.markdown(
-        f"<div style='text-align: justify;'><strong>{category}:</strong> {variables_str}</div>",
-        unsafe_allow_html=True,
-    )
-
-# Store final dataframe and bin dictionary into session state
-st.session_state["final_df"], st.session_state["bin_dict"] = final_df, category_dict
-
-if st.button('Save Changes'):
-    
-    with open("Pickle_files/main_df", 'wb') as f:
-        pickle.dump(st.session_state["final_df"], f)
-    with open("Pickle_files/category_dict",'wb') as c:
-        pickle.dump(st.session_state["bin_dict"],c)
-    st.success('Changes Saved!')
-
-
-

pages/Help.py

@@ -0,0 +1,204 @@
+# Importing necessary libraries
+import streamlit as st
+
+st.set_page_config(
+    page_title="Model Build",
+    page_icon=":shark:",
+    layout="wide",
+    initial_sidebar_state="collapsed",
+)
+
+import numpy as np
+import pandas as pd
+from utilities import set_header, load_local_css, load_authenticator
+import pickle
+import base64
+
+load_local_css("styles.css")
+set_header()
+
+
+st.header('MASTERCARD MMO TOOL')
+
+st.subheader('Overview')
+
+st.markdown('The tool was developed in accordance with the best practices for building Marketing mix models for different clients and businesses. The tool can be used for building various MMM models, optimize spends and execute various simulations. ')
+
+st.markdown('Last Updated: 3/26/204')
+user=st.radio("Select User",['Data Scientist','Media Planner'],horizontal=True,index=1)
+
+
+
+if user=='Data Scientist':
+
+    with st.expander('**Data Import**'):
+
+        st.markdown("""
+
+        The Data Import page allows users to bring in any additional data apart from what’s being fetched using APIs and processed by Data Engineering pipelines and standardize both API and non-API data to the desired end granularity. It features options for feature engineering, variable grouping, and data imputation. Additionally, it provides a comprehensive summary of all actions performed on the page, ensuring a transparent and efficient data preparation process.
+
+        **Features:**
+        - **Categorization:** Allows for the categorization of similar variables for streamlined analysis.
+        - **Feature Engineering:** Enables the creation and grouping of columns to build better models.
+        - **Data Imputation:** Provides methods to fill in missing or incomplete data values.
+        """)
+
+    with st.expander('**Data Validation**'):
+
+        st.markdown("""
+
+        This page is designed to enhance data quality and insights, focusing on selected targets and panels. It offers Response Metric Analysis, Univariate and Bivariate Report, alongside Media and Non-Media Variables Analysis. Correlation is explored to ensure a thorough validation process.
+
+        **Features:**
+        - **Response Metric Analysis:** Evaluates the performance metrics and trends relevant to the selected target and panel.
+        - **Univariate and Bivariate Report:** Offers a detailed univariate and bivariate report.
+        - **Variables Analysis:** Evaluates the significance of media and non-media variables for the chosen target/panel and validates the variables to be considered for the next steps.
+        - **Correlation Analysis:** Utilizes correlation plots to reveal the relationships between variables.
+        """)
+
+    with st.expander('**Transformation**'):
+        
+
+        st.markdown("""
+
+        Transformation capabilities include Media and Exogenous Transformations such as lag, lead, moving average, power, saturation, and adstock adjustments. This page not only applies these transformations but also summarizes the process and the order of operations, providing clarity on the data's manipulation.
+
+        **Features:**
+        - **Transformations:** Applies specific alterations (lag, lead, moving average, power, saturation, and adstock) to media and exogenous variables to enhance model performance.
+        - **Summarization of Transformation:** Provides a detailed overview of all transformations applied, including their sequence.
+        """)
+
+    with st.expander('**Build Model**'):
+
+        st.markdown("""
+
+        This feature integrates transformation creation with model building for both panel and non-panel levels. It streamlines the process, making it easier for users to construct and refine their models based on the transformed data. After building models, this page assists in selecting the best fit and provides a detailed summary. It includes comparisons of Actual vs. Predicted outcomes, Residual Analysis, and Variance Inflation Factor (VIF) for both test and train datasets.
+
+        **Features:**
+        - **Diverse Models:** Constructs models for various metrics using OLS and mixed effect models.
+        - **Model Selection:** Chooses the most significant model utilizing metrics such as coefficients, P-value, R Square, Adjusted R Square, and MAPE.
+        """)
+
+    with st.expander('**Model Tuning**'):
+
+        st.markdown("""
+        ###
+        Model Tuning offers advanced options like Event Flags, addition of Trends, Cyclical Pattern, sine and cosine waves. These features help in refining the model by accounting for specific events, trends, and seasonal patterns.
+
+        **Features:**
+        - **Event Flags:** Incorporates the effect of specific events on the target.
+        - **Trends:** Incorporates long-term trends and seasonality.
+        - **Cyclical Pattern:** Utilizes sine and cosine waves to capture and adjust for seasonal variations.
+        - **Contributions Analysis: Calculates contributions from the tuned model for each media channel
+        """)
+
+    with st.expander("**Save Model Results**"):
+        
+        st.markdown("""
+
+        This page saves the model's outcomes, including channel contributions, an EDA report, and a thorough Analysis of Model Results. It's designed to document and preserve the work done for future reference.
+
+        **Features:**
+        - **Channel Contribution:** Details the impact of each media channel on outcomes.
+        - **EDA Report:** Provides an exploratory data analysis summary.
+        - **Analysis of Model Results:** Offers a comprehensive review of the model's performance.
+        """)
+
+    with st.expander('**Model Results Overview**'):
+
+        st.markdown("""
+
+        This section provides a comprehensive overview of historical spending, including channel-wise spends, revenue, ROI, and weekly contributions. It also details channel spends and revenue on a week-by-week basis, offering a granular look at financial performance.
+
+        **Features:**
+        - **Spends Analysis:** Breaks down channel-wise spend and revenue.
+        - **ROI and Contributions:** Evaluates return on investment and weekly/aggregated channel performance.
+        """)
+
+    with st.expander('**Build Response Curves**'):
+
+        st.markdown("""
+                    
+        This page updates response curves and allows for testing and saving these fits. It's essential for understanding how different levels of spending affect outcomes and for refining marketing strategies.
+
+        **Features:**
+        - **Response Curve Update:** Allows for the modification and refinement of response curves.
+        - **Curve Testing and Saving:** Facilitates the evaluation of curve fits and preserves/download curve parameters.
+        """)
+
+    with st.expander('**Scenario Planner**'):
+
+        st.markdown("""
+
+        The Scenario Planner page enables forward and reverse optimization, allowing users to either maximize targets given certain spends or minimize spends given a target revenue. It includes responsive S curves for multiple channels and optimizes them accordingly.
+
+        **Features:**
+        - **Optimization:** Supports both forward and reverse financial planning, with adjustments based on percentage or actual spend values.
+        - **Channel Focused:** Enables optimization based on specific media channels for focused strategies.
+        - **Responsive S Curves:** Showcases real-time, responsive S curves that highlight regions indicating underinvestment, optimal spending, and areas of potential overinvestment.
+        - **Dynamic Visualization:** Provides ROI/MROI for in-depth analysis and immediate scenario feedback.
+        """)
+    with st.expander("**Saved Scenarios**"):
+
+        st.markdown("""
+      
+        Users can save, load, download, and delete scenarios involving spends, ROI, and MROI for both actual and simulated cases. This feature offers flexibility in scenario management and comparison.
+
+        **Features:**
+        - **Optimized Results Visualization:** Displays the outcomes of optimization, highlighting effectiveness and efficiency in scenario planning.
+        - **Effectiveness and Efficiency Analysis:** Provides detailed visual insights into how optimization impacts campaign performance and resource utilization.
+        """)
+
+    with st.expander("**Optimized Result Analysis**"):
+
+        st.markdown("""
+                    
+        This analysis page gives an overview of optimized spends (actual, planned, and delta), budget allocation (% split and planned spends), and forecasts on response and return by media channel. It's designed to provide insights into the efficiency and effectiveness of optimized media spending strategies.
+
+        **Features:**
+        - **Optimized Spends Overview:** Compares actual, planned, and delta spends.
+        - **Budget Allocation Analysis:** Breaks down the percentage split and planned expenditures.
+        """)
+
+if user=='Media Planner':
+
+    with st.expander('**Scenario Planner**'):
+
+        st.markdown("""
+
+        The Scenario Planner page enables forward and reverse optimization, allowing users to either maximize targets given certain spends or minimize spends given a target revenue. It includes responsive S curves for multiple channels and optimizes them accordingly.
+
+        **Features:**
+        - **Optimization:** Supports both forward and reverse financial planning, with adjustments based on percentage or actual spend values.
+        - **Channel Focused:** Enables optimization based on specific media channels for focused strategies.
+        - **Responsive S Curves:** Showcases real-time, responsive S curves that highlight regions indicating underinvestment, optimal spending, and areas of potential overinvestment.
+        - **Dynamic Visualization:** Provides ROI/MROI for in-depth analysis and immediate scenario feedback.
+        """)
+    with st.expander("**Saved Scenario**"):
+
+        st.markdown("""
+      
+        Users can save, load, download, and delete scenarios involving spends, ROI, and MROI for both actual and simulated cases. This feature offers flexibility in scenario management and comparison.
+
+        **Features:**
+        - **Optimized Results Visualization:** Displays the outcomes of optimization, highlighting effectiveness and efficiency in scenario planning.
+        - **Effectiveness and Efficiency Analysis:** Provides detailed visual insights into how optimization impacts campaign performance and resource utilization.
+        """)
+
+    with st.expander("**Optimized Result Analysis**"):
+
+        st.markdown("""
+                    
+        This analysis page gives an overview of optimized spends (actual, planned, and delta), budget allocation (% split and planned spends), and forecasts on response and return by media channel. It's designed to provide insights into the efficiency and effectiveness of optimized media spending strategies.
+
+        **Features:**
+        - **Optimized Spends Overview:** Compares actual, planned, and delta spends.
+        - **Budget Allocation Analysis:** Breaks down the percentage split and planned expenditures.
+        """)
+
+
+with open("MMM Tool Description.docx", "rb") as file:
+    word_content = file.read()
+    b64 = base64.b64encode(word_content).decode()
+    href = f'<a href="data:application/octet-stream;base64,{b64}" download="document.docx">Download Document</a>'
+    st.markdown(href, unsafe_allow_html=True)