Code Organization and Modularity
Issue:
The current implementation includes a vast number of helper functions within a single script, which can reduce readability and maintainability.

Recommendation:

Modularization: Separate helper functions, utilities, and main Streamlit app into different Python modules/files.

utils.py for utility functions (e.g., data fetching, calculations).

plots.py for all plotting-related functions.

metrics.py for financial metric calculations.

app.py for the main Streamlit application logic.

Example Structure:
portfolio_optimizer/
├── app.py
├── utils.py
├── plots.py
├── metrics.py
└── requirements.txt
Benefits:

Enhanced readability.

Easier debugging and testing.

Reusability of components across different parts of the application.

tell me in detail how can I do it, step by step, Please review my code and provide the original code alongside the updated one. Include the exact instructions for where and how to apply the corrections in the original code by referencing specific function names, variable names, or line numbers if possible. Dont give me general corrections as I am a newbie in coding, so make your response concise and simple, showing only the revised code snippets with a brief explanation for each change.

%%writefile portfolio_optimizer.py
import streamlit as st
import pandas as pd
import plotly.graph_objects as go
import numpy as np
import yfinance as yf
import statsmodels.api as sm
import datetime
from datetime import datetime, timedelta
from pandas_datareader import data as pdr
import plotly.express as px
import warnings
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from scipy.stats import skew, kurtosis
from joblib import Parallel, delayed
import json
from functools import lru_cache
from concurrent.futures import ProcessPoolExecutor
from scipy.optimize import minimize
from scipy.stats import norm, skew, kurtosis
from requests.packages.urllib3.exceptions import InsecureRequestWarning

warnings.filterwarnings('ignore')

# ----------------------------
# Frequency Mapping
# ----------------------------
frequency_mapping = {
    "Daily": "D",
    "Weekly": "W",
    "Monthly": "M",
    "Quarterly": "Q",
    "Yearly": "Y"
}

# ----------------------------
# Helper Functions
# ----------------------------

def get_company_name(ticker_df, ticker):
    match = ticker_df[ticker_df['Ticker'] == ticker]
    if not match.empty:
        return match.iloc[0]['Company Name']
    return "Unknown"

def format_asset_option(ticker, company_name):
    return f"{ticker} - {company_name}"
    
# Global SSL bypass setup for requests
# Suppress only the InsecureRequestWarning, if you are bypassing SSL
warnings.simplefilter('ignore', InsecureRequestWarning)
requests.packages.urllib3.disable_warnings()  # Suppresses SSL warnings (use cautiously)

def calculate_risk_factor_attribution(returns, factors, annualization_factor=252):
    """
    Decompose portfolio returns by risk factors using linear regression and provide
    comments on the factor contributions.

    Parameters:
    - returns (pd.Series): Portfolio returns.
    - factors (pd.DataFrame): DataFrame where each column represents a risk factor.
    - annualization_factor (int): Annualization factor (252 for daily, 12 for monthly returns).

    Returns:
    - pd.DataFrame: Factor names, their contribution percentages, and explanatory comments.
    - str: Error message if any exception occurs, else None.
    """
    try:
        # Ensure returns has a name, or assign one if missing
        if returns.name is None:
            returns = returns.rename("Portfolio_Returns")
        
        # Convert returns to a DataFrame
        returns = returns.to_frame()
        
        # Ensure both returns and factors are in decimal format
        if returns['Portfolio_Returns'].max() > 1:
            returns = returns / 100
            
        if factors.max().max() > 1:
            factors = factors / 100
        
        # Adjust returns to excess returns by subtracting RF (if it exists in factors)
        if 'RF' in factors.columns:
            returns['Portfolio_Returns'] = returns['Portfolio_Returns'] - factors['RF']
            factors = factors.drop(columns=['RF'])
        
        # Align the data using an inner join and drop any remaining NaN values
        aligned_data = pd.merge(returns, factors, left_index=True, right_index=True, how='inner').dropna()

        # Check that we have data after merging
        if aligned_data.empty:
            raise ValueError("No overlapping dates between returns and factors after dropping NaNs.")
    
        # Prepare X and y for regression
        X = aligned_data[factors.columns]
        X = sm.add_constant(X)  # Add intercept
        y = aligned_data['Portfolio_Returns']
        
        # Run the regression model
        model = sm.OLS(y, X).fit()
        coefficients = model.params.drop('const', errors='ignore')
        
        # Calculate factor contributions (non-annualized)
        factor_means = X.mean().drop('const', errors='ignore')
        contributions = coefficients * factor_means
        
        # Calculate total return
        total_return = returns['Portfolio_Returns'].mean()

        # Check if total_return is zero to avoid division by zero
        if np.isclose(total_return, 0):
            st.warning("Total portfolio return is zero. Cannot compute contribution percentages.")
            attribution_df = pd.DataFrame({
                'Factor': contributions.index,
                'Contribution (%)': [0.0] * len(contributions)
            })
        else:
            # Calculate percentage contributions
            attribution_df = pd.DataFrame({
                'Factor': contributions.index,
                'Contribution (%)': (contributions / total_return * 100).round(2)
            })
        
        # Remove the constant term if it exists
        if 'const' in attribution_df['Factor'].values:
            attribution_df = attribution_df[attribution_df['Factor'] != 'const']
        
        # Use full names for factors
        factor_full_names = {
            'Mkt-RF': 'Market Risk Premium',
            'SMB': 'Small Minus Big',
            'HML': 'High Minus Low'
        }
        attribution_df['Factor'] = attribution_df['Factor'].replace(factor_full_names)
        
        # Generate detailed comments based on factor contributions
        comments = []
        for _, row in attribution_df.iterrows():
            factor_name = row['Factor']
            contribution = row['Contribution (%)']

            # Market Risk Premium (Mkt-RF) Analysis
            if factor_name == 'Market Risk Premium':
                if contribution > 75:
                    comments.append(f"{factor_name} = {contribution}%: Extremely high sensitivity to market movements.")
                elif contribution > 50:
                    comments.append(f"{factor_name} = {contribution}%: High sensitivity to overall market movements.")
                elif contribution > 30:
                    comments.append(f"{factor_name} = {contribution}%: Notable sensitivity to market risk.")
                elif contribution > 10:
                    comments.append(f"{factor_name} = {contribution}%: Moderate exposure to market risk.")
                else:
                    comments.append(f"{factor_name} = {contribution}%: Low exposure to market risk.")

            # Small Minus Big (SMB) Analysis
            elif factor_name == 'Small Minus Big':
                if contribution > 5:
                    comments.append(f"{factor_name} = {contribution}%: Strong tilt towards small-cap stocks.")
                elif contribution > 0:
                    comments.append(f"{factor_name} = {contribution}%: Mild preference for small-cap stocks.")
                elif contribution > -5:
                    comments.append(f"{factor_name} = {contribution}%: Slight tilt towards larger-cap stocks.")
                else:
                    comments.append(f"{factor_name} = {contribution}%: Clear preference for large-cap stocks.")

            # High Minus Low (HML) Analysis
            elif factor_name == 'High Minus Low':
                if contribution > 5:
                    comments.append(f"{factor_name} = {contribution}%: Strong tilt toward value stocks.")
                elif contribution > 0:
                    comments.append(f"{factor_name} = {contribution}%: Slight positive tilt toward value stocks.")
                elif contribution > -5:
                    comments.append(f"{factor_name} = {contribution}%: Minor negative tilt toward growth stocks.")
                else:
                    comments.append(f"{factor_name} = {contribution}%: Clear preference for growth stocks.")

        # Add comments to the DataFrame
        attribution_df['Comment'] = comments
    
        return attribution_df, None
    
    except Exception as e:
        error_message = f"Error calculating risk factor attribution: {e}"
        return pd.DataFrame(), error_message

# Fetch actual factor data from Fama-French or another data source
def fetch_fama_french_factors(start_date, end_date):
    try:
        # Fetch daily Fama-French factors
        ff_data = pdr.get_data_famafrench('F-F_Research_Data_Factors_daily', start=start_date, end=end_date)
        if not ff_data:
            st.error("Fama-French data not found.")
            return pd.DataFrame()
        
        factors = ff_data[0]
        factors = factors.rename(columns=lambda x: x.strip())  # Remove any leading/trailing spaces
        factors /= 100  # Convert percentage returns to decimal format
        
        # Fill missing data by forward and backward filling
        factors = factors.fillna(method='ffill').fillna(method='bfill')
        
        # Check for necessary columns
        required_columns = {'Mkt-RF', 'SMB', 'HML', 'RF'}
        if not required_columns.issubset(factors.columns):
            missing = required_columns - set(factors.columns)
            st.error(f"Missing factor columns: {', '.join(missing)}")
            return pd.DataFrame()
        
        # Removed debugging outputs
        # st.write("Fama-French factors fetched successfully:")
        # st.dataframe(factors.head())
        
        return factors
    except Exception as e:
        st.error(f"Error fetching factor data: {e}")
        return pd.DataFrame()

def calculate_final_score(primary_metrics, comparison_metrics):
    metrics_to_compare = [
        'Annualized Return (CAGR)',
        'Sharpe Ratio',
        'Sortino Ratio',
        'Treynor Ratio',
        'Calmar Ratio',
        'Alpha (annualized)',
        'Information Ratio',
        'Modigliani–Modigliani Measure',
        'Upside Capture Ratio',
        'Gain/Loss Ratio'
    ]
    metric_weights = {
        'Annualized Return (CAGR)': 15,
        'Sharpe Ratio': 15,
        'Sortino Ratio': 10,
        'Treynor Ratio': 10,
        'Calmar Ratio': 10,
        'Alpha (annualized)': 10,
        'Information Ratio': 10,
        'Modigliani–Modigliani Measure': 10,
        'Upside Capture Ratio': 5,
        'Gain/Loss Ratio': 5
    }

    score = 0
    total = 0

    for metric in metrics_to_compare:
        primary = primary_metrics.get(metric, "N/A")
        comparison = comparison_metrics.get(metric, "N/A")
        if primary != "N/A" and comparison != "N/A":
            primary_val = float(primary.strip('%')) if isinstance(primary, str) and '%' in primary else float(primary)
            comparison_val = float(comparison.strip('%')) if isinstance(comparison, str) and '%' in comparison else float(comparison)
            weight = metric_weights.get(metric, 1)
            if comparison_val == 0:
                st.warning(f"Comparison metric '{metric}' has a value of zero. Skipping this metric to avoid division by zero.")
                continue
            if primary_val > comparison_val:
                score += weight * (primary_val / comparison_val)
            else:
                score += weight * (primary_val / comparison_val) * 0.5  # Partial credit
            total += weight
    return (score / total) * 100 if total > 0 else 0

def plot_growth_comparison(cum_returns, benchmark_cum_returns):
    try:
        # Align the indices to ensure matching dates
        common_index = cum_returns.index.intersection(benchmark_cum_returns.index)
        cum_returns = cum_returns.loc[common_index]
        benchmark_cum_returns = benchmark_cum_returns.loc[common_index]

        if cum_returns.empty or benchmark_cum_returns.empty:
            st.warning("No overlapping data to plot growth comparison.")
            return

        df = pd.DataFrame({
            'Date': cum_returns.index,
            'Portfolio': cum_returns.values,
            'Benchmark': benchmark_cum_returns.values
        })

        fig = px.line(
            df,
            x='Date',
            y=['Portfolio', 'Benchmark'],
            title='Growth Comparison',
            labels={'value': 'Cumulative Returns', 'Date': 'Date'},
            hover_data={'Date': '|%B %d, %Y'},  # Enhanced hover format
            template='plotly_dark'  # Use dark template for better contrast
        )
        fig.update_layout(
            hovermode='x unified',
            xaxis=dict(rangeslider=dict(visible=True), type='date'),
        )
        fig.update_traces(line=dict(width=2))
        st.plotly_chart(fig, use_container_width=True, config={
            'scrollZoom': True
        })
    except Exception as e:
        st.error(f"Error plotting growth comparison: {e}")

def plot_drawdown_comparison(drawdown_portfolio, drawdown_benchmark):
    try:
        df = pd.DataFrame({
            'Portfolio Drawdown': drawdown_portfolio,
            'Benchmark Drawdown': drawdown_benchmark
        })
        fig = px.line(df, title='Drawdown Comparison', labels={'value': 'Drawdown (%)', 'index': 'Date'}, template='plotly_dark')
        fig.update_layout(
            hovermode='x unified',
            xaxis=dict(rangeslider=dict(visible=True), type='date'),
        )
        fig.update_traces(line=dict(width=2))
        st.plotly_chart(fig, use_container_width=True, config={
            'scrollZoom': True})
    except Exception as e:
        st.error(f"Error plotting drawdown comparison: {e}")
        
def plot_cagr_over_time(cum_returns, time_frames=['Weekly', 'Monthly', 'Quarterly', 'Annually']):
    try:
        frequency_map = {
            'Weekly': 'W',
            'Monthly': 'M',
            'Quarterly': 'Q',
            'Annually': 'Y'
        }
        fig = go.Figure()
        for tf in time_frames:
            freq = frequency_map.get(tf)
            if not freq:
                continue
            rolled = cum_returns.resample(freq).last()
            years = (rolled.index[-1] - rolled.index[0]).days / 365.25
            if years <= 0:
                st.warning(f"Not enough data to calculate {tf} CAGR.")
                continue
            cagr = (rolled / rolled.iloc[0]) ** (1/years) - 1
            fig.add_trace(go.Scatter(x=rolled.index, y=cagr, mode='lines', name=f'{tf} CAGR'))
        fig.update_layout(
            title='CAGR Over Multiple Time Frames',
            xaxis_title='Date',
            yaxis_title='CAGR',
            hovermode='x unified'
        )
        st.plotly_chart(fig, use_container_width=True)
    except Exception as e:
        st.error(f"Error plotting CAGR over time: {e}")

def backtest(weights, prices, rebalance_freq='M', broker_fee=0.0, debug=False):
    try:
        # Calculate returns and drop any NaN values
        returns = prices.pct_change().dropna()
        if returns.empty:
            st.error("Returns data is empty after calculating percentage changes.")
            return pd.Series(dtype=float), pd.Series(dtype=float)
        
        # Calculate portfolio returns
        portfolio_returns = returns.dot(weights)
        if not isinstance(portfolio_returns, pd.Series):
            portfolio_returns = portfolio_returns.squeeze()

        # Identify rebalancing dates
        if rebalance_freq == 'D':
            rebalance_dates = returns.index
        else:
            rebalance_dates = returns.resample(rebalance_freq).last().dropna().index

        # Debug: Show rebalancing dates
        if debug:
            st.write(f"Rebalancing Dates: {rebalance_dates.tolist()}")

        # Ensure rebalance_dates are in the portfolio_returns index
        valid_rebalance_dates = rebalance_dates.intersection(portfolio_returns.index)
        if debug:
            st.write(f"Valid Rebalancing Dates: {valid_rebalance_dates.tolist()}")

        # Apply broker fees on rebalancing dates
        if not valid_rebalance_dates.empty:
            portfolio_returns.loc[valid_rebalance_dates] -= broker_fee / 100  # Convert to decimal
        else:
            st.warning("No valid rebalancing dates found within the returns data.")

        # Calculate cumulative returns
        cum_returns = (1 + portfolio_returns).cumprod()
        if cum_returns.empty:
            st.error("Cumulative returns are empty. Check the data and allocations.")
            return pd.Series(dtype=float), pd.Series(dtype=float)

        # Debug: Show cumulative returns
        if debug:
            st.write("Cumulative Returns:")
            st.write(cum_returns)

        return portfolio_returns, cum_returns
    except Exception as e:
        st.error(f"Error during backtesting: {e}")
        return pd.Series(dtype=float), pd.Series(dtype=float)

def calculate_simple_metrics(returns, cum_returns, start_balance=10000):
    metrics = {}
    metrics['Start Balance'] = f"${start_balance:,.2f}"
    if not cum_returns.empty:
        try:
            end_balance = start_balance * cum_returns.iloc[-1]
            metrics['End Balance'] = f"${end_balance:,.2f}"
            start_date = cum_returns.index[0]
            end_date = cum_returns.index[-1]
            days = (end_date - start_date).days
            years = days / 365.25
            cagr = (end_balance / start_balance) ** (1 / years) - 1 if years > 0 else np.nan
            metrics['Annualized Return (CAGR)'] = f"{cagr * 100:.2f}%" if not np.isnan(cagr) else "N/A"
            yearly_returns = returns.resample('Y').apply(lambda x: (1 + x).prod() - 1)
            best_year = yearly_returns.max() * 100 if not yearly_returns.empty else np.nan
            worst_year = yearly_returns.min() * 100 if not yearly_returns.empty else np.nan
            metrics['Best Year'] = f"{best_year:.2f}%" if not np.isnan(best_year) else "N/A"
            metrics['Worst Year'] = f"{worst_year:.2f}%" if not np.isnan(worst_year) else "N/A"
            metrics['Arithmetic Mean (Monthly)'] = f"{returns.mean() * 100:.2f}%" if not returns.empty else np.nan
            metrics['Arithmetic Mean (Annualized)'] = f"{returns.mean() * 252 * 100:.2f}%" if not returns.empty else np.nan
            metrics['Geometric Mean (Monthly)'] = f"{(np.exp(np.log1p(returns).mean()) - 1) * 100:.2f}%" if not returns.empty else np.nan
            metrics['Geometric Mean (Annualized)'] = f"{(np.exp(np.log1p(returns).mean() * 252) - 1) * 100:.2f}%" if not returns.empty else np.nan
            metrics['Standard Deviation (Monthly)'] = f"{returns.std() * 100:.2f}%" if not returns.empty else np.nan
            metrics['Standard Deviation (Annualized)'] = f"{returns.std() * np.sqrt(252) * 100:.2f}%" if not returns.empty else np.nan
            metrics['Downside Deviation (Monthly)'] = f"{returns[returns < 0].std() * 100:.2f}%" if not returns.empty else np.nan
            metrics['Maximum Drawdown'] = f"{drawdown(cum_returns) * 100:.2f}%" if not cum_returns.empty else np.nan
            metrics['Sharpe Ratio'] = f"{calculate_sharpe_ratio(returns, 0.02):.2f}" if not cum_returns.empty else np.nan
            metrics['Sortino Ratio'] = f"{calculate_sortino_ratio(returns, 0.02):.2f}" if not cum_returns.empty else np.nan
            metrics['Gain/Loss Ratio'] = f"{calculate_gain_loss_ratio(returns):.2f}" if not cum_returns.empty else np.nan
            metrics['Skewness'] = f"{skew(returns):.2f}" if not returns.empty else np.nan
            metrics['Excess Kurtosis'] = f"{kurtosis(returns):.2f}" if not returns.empty else np.nan
            metrics['Safe Withdrawal Rate'] = f"{calculate_safe_withdrawal_rate(returns):.6f}%" if not returns.empty else np.nan
            metrics['Perpetual Withdrawal Rate'] = f"{calculate_perpetual_withdrawal_rate(returns):.6f}%" if not returns.empty else np.nan
            metrics['Positive Periods'] = calculate_positive_periods(returns) if not returns.empty else "N/A"
        except Exception as e:
            st.error(f"Error calculating simple metrics: {e}")
            for key in ['End Balance', 'Annualized Return (CAGR)', 'Best Year', 'Worst Year',
                        'Arithmetic Mean (Monthly)', 'Arithmetic Mean (Annualized)',
                        'Geometric Mean (Monthly)', 'Geometric Mean (Annualized)',
                        'Standard Deviation (Monthly)', 'Standard Deviation (Annualized)',
                        'Downside Deviation (Monthly)', 'Maximum Drawdown',
                        'Sharpe Ratio', 'Sortino Ratio', 'Gain/Loss Ratio',
                        'Skewness', 'Excess Kurtosis', 'Safe Withdrawal Rate',
                        'Perpetual Withdrawal Rate', 'Positive Periods']:
                metrics[key] = np.nan  # Use NaN instead of "N/A"
    else:
        for key in ['End Balance', 'Annualized Return (CAGR)', 'Best Year', 'Worst Year',
                    'Arithmetic Mean (Monthly)', 'Arithmetic Mean (Annualized)',
                    'Geometric Mean (Monthly)', 'Geometric Mean (Annualized)',
                    'Standard Deviation (Monthly)', 'Standard Deviation (Annualized)',
                    'Downside Deviation (Monthly)', 'Maximum Drawdown',
                    'Sharpe Ratio', 'Sortino Ratio', 'Gain/Loss Ratio',
                    'Skewness', 'Excess Kurtosis', 'Safe Withdrawal Rate',
                    'Perpetual Withdrawal Rate', 'Positive Periods']:
            metrics[key] = np.nan  # Use NaN instead of "N/A"
    
    return metrics

def calculate_sharpe_ratio(returns, rf=0.02):
    """
    Calculate the Sharpe Ratio for a given set of returns.

    Parameters:
    - returns (pd.Series): Daily returns of the portfolio.
    - rf (float): Risk-free rate (default is 2%).

    Returns:
    - float: Sharpe Ratio.
    """
    excess_return = returns.mean() * 252 - rf
    std_dev = returns.std() * np.sqrt(252)
    return excess_return / std_dev if std_dev != 0 else np.nan

def calculate_sortino_ratio(returns, rf=0.02):
    excess_return = returns.mean() * 252 - rf
    downside_std = returns[returns < 0].std() * np.sqrt(252)
    return excess_return / downside_std if downside_std != 0 else np.nan

def calculate_treynor_ratio(returns, benchmark_returns, rf=0.02):
    beta = calculate_beta(returns, benchmark_returns)
    excess_return = returns.mean() * 252 - rf
    return excess_return / beta if beta != 0 else np.nan

def calculate_calmar_ratio(returns, cum_returns):
    annual_return = returns.mean() * 252
    max_dd = drawdown(cum_returns)
    return annual_return / abs(max_dd) if max_dd != 0 else np.nan

def calculate_beta(returns, benchmark_returns):
    if returns.empty or benchmark_returns.empty:
        return np.nan
    covariance_matrix = np.cov(returns, benchmark_returns)
    covariance = covariance_matrix[0, 1]
    benchmark_variance = covariance_matrix[1, 1]
    return covariance / benchmark_variance if benchmark_variance != 0 else np.nan

def calculate_alpha(returns, benchmark_returns, rf=0.02):
    beta = calculate_beta(returns, benchmark_returns)
    portfolio_return = returns.mean() * 252
    benchmark_return = benchmark_returns.mean() * 252
    return portfolio_return - (rf + beta * (benchmark_return - rf)) if not np.isnan(beta) else np.nan

def calculate_r_squared(returns, benchmark_returns):
    if returns.empty or benchmark_returns.empty:
        return np.nan
    covariance = np.cov(returns, benchmark_returns)
    var_port = covariance[0,0]
    var_bench = covariance[1,1]
    cov = covariance[0,1]
    return (cov ** 2) / (var_port * var_bench) if var_port !=0 and var_bench !=0 else np.nan

def calculate_information_ratio(returns, benchmark_returns):
    if returns.empty or benchmark_returns.empty:
        return np.nan
    active_return = (returns.mean() - benchmark_returns.mean()) * 252
    tracking_error = calculate_tracking_error(returns, benchmark_returns)
    return active_return / tracking_error if tracking_error != 0 else np.nan

def generate_recommendations(allocations, available_selected):
    """
    Generate basic portfolio recommendations based on current allocations.

    Parameters:
    - allocations (list of float): Current allocation percentages.
    - available_selected (list of str): List of selected asset tickers.

    Returns:
    - list of str: Recommendations
    """
    recommendations = []
    max_alloc = max(allocations)
    min_alloc = min(allocations)
    total_alloc = sum(allocations)
    
    if max_alloc > 40.0:
        idx = allocations.index(max_alloc)
        ticker = available_selected[idx]
        recommendations.append(f"🔹 Consider reducing the allocation to **{ticker}** since it constitutes **{max_alloc:.2f}%** of your portfolio.")
    
    if min_alloc < 5.0:
        idx = allocations.index(min_alloc)
        ticker = available_selected[idx]
        recommendations.append(f"🔸 Consider increasing the allocation to **{ticker}** to at least **5.00%** for better diversification.")
    
    if total_alloc < 100.0:
        recommendations.append("🔹 Consider allocating the remaining funds to additional assets to reach a total of **100%**.")
    
    return recommendations

def get_common_benchmarks(selected_tickers):
    # Define benchmark suggestions based on asset sectors or indices
    sp500 = {"AAPL", "MSFT", "GOOGL", "AMZN", "META", "TSLA"}  # Example S&P 500 tech companies
    nasdaq_tech = {"AAPL", "MSFT", "GOOGL", "AMZN", "META", "TSLA", "NVDA", "AMD"}

    selected_set = set(selected_tickers)

    if selected_set.issubset(sp500):
        return {
            "S&P 500": "^GSPC",
            "Dow Jones Industrial Average": "^DJI",
            "Russell 2000": "^RUT",
            "Custom": "CUSTOM"
        }
    elif selected_set.issubset(nasdaq_tech):
        return {
            "NASDAQ Composite": "^IXIC",
            "QQQ (Invesco QQQ ETF)": "QQQ",
            "Custom": "CUSTOM"
        }
    else:
        return {
            "S&P 500": "^GSPC",
            "NASDAQ Composite": "^IXIC",
            "Dow Jones Industrial Average": "^DJI",
            "Russell 2000": "^RUT",
            "Custom": "CUSTOM"
        }

def calculate_tracking_error(returns, benchmark_returns):
    if returns.empty or benchmark_returns.empty:
        return np.nan
    return np.std((returns - benchmark_returns)) * np.sqrt(252)

def calculate_performance_attribution(returns, weights):
    try:
        if isinstance(returns, pd.Series):
            returns = returns.to_frame('Asset')  # Convert Series to DataFrame with a default column name
        
        if len(weights) != len(returns.columns):
            st.error(f"Number of weights ({len(weights)}) does not match number of assets ({len(returns.columns)}).")
            return pd.DataFrame()
        
        annual_returns = returns.mean() * 252
        contributions = annual_returns * weights
        attribution_df = pd.DataFrame({
            'Asset': returns.columns,
            'Contribution (%)': (contributions / contributions.sum() * 100).round(2)
        }).sort_values(by='Contribution (%)', ascending=False)
        return attribution_df
    except Exception as e:
        st.error(f"Error calculating performance attribution: {e}")
        return pd.DataFrame()
        
def calculate_active_return(returns, benchmark_returns):
    if returns.empty or benchmark_returns.empty:
        return np.nan
    return (returns.mean() - benchmark_returns.mean()) * 252 * 100

def calculate_gain_loss_ratio(returns):
    gains = returns[returns > 0].sum()
    losses = -returns[returns < 0].sum()
    return gains / losses if losses != 0 else np.nan

def drawdown(cum_returns):
    if cum_returns.empty:
        return np.nan
    peak = cum_returns.expanding(min_periods=1).max()
    dd = (cum_returns / peak) - 1
    return dd.min()

def calculate_capture_ratio(returns, benchmark_returns, upside=True):
    if returns.empty or benchmark_returns.empty:
        return np.nan
    mask = benchmark_returns > 0 if upside else benchmark_returns < 0
    if mask.sum() == 0:
        return np.nan
    portfolio = returns[mask]
    benchmark = benchmark_returns[mask]
    return (portfolio.sum() / benchmark.sum()) * 100 if benchmark.sum() != 0 else np.nan

def calculate_safe_withdrawal_rate(returns):
    if returns.empty:
        return np.nan
    return (returns.mean() / returns.std()) * 100

def calculate_perpetual_withdrawal_rate(returns):
    if returns.empty:
        return np.nan
    return (returns.mean() / returns.std()) * 100

def calculate_positive_periods(returns):
    if returns.empty:
        return "N/A"
    positive = returns > 0
    return f"{positive.sum()} out of {len(returns)} ({(positive.sum()/len(returns))*100:.2f}%)"

def calculate_modigliani_miller(returns, benchmark_returns, rf=0.02):
    sharpe = calculate_sharpe_ratio(returns, rf)
    alpha = calculate_alpha(returns, benchmark_returns, rf)
    return alpha / sharpe if sharpe != 0 and not np.isnan(alpha) else np.nan

def get_drawdown_details(cum_returns):
    drawdowns = []
    if cum_returns.empty:
        return drawdowns
    peak = cum_returns.iloc[0]
    peak_date = cum_returns.index[0]
    trough = cum_returns.iloc[0]
    trough_date = cum_returns.index[0]
    for date, value in cum_returns.items():
        if value > peak:
            if trough < peak:
                recovery = cum_returns[cum_returns >= peak].loc[trough_date:]
                if not recovery.empty:
                    recovery_date = recovery.index[0]
                    if recovery_date > trough_date:
                        recovery_time = (recovery_date - trough_date).days
                        underwater_period = (recovery_date - peak_date).days
                        drawdowns.append({
                            'Start': peak_date.strftime('%b %Y'),
                            'End': trough_date.strftime('%b %Y'),
                            'Length': f"{(trough_date - peak_date).days // 30} months",
                            'Recovery By': recovery_date.strftime('%b %Y'),
                            'Recovery Time': f"{recovery_time // 30} months",
                            'Underwater Period': f"{underwater_period // 30} months",
                            'Drawdown': f"{((trough / peak) - 1) * 100:.2f}%"
                        })
                    else:
                        drawdowns.append({
                            'Start': peak_date.strftime('%b %Y'),
                            'End': trough_date.strftime('%b %Y'),
                            'Length': f"{(trough_date - peak_date).days // 30} months",
                            'Recovery By': "Not Recovered",
                            'Recovery Time': "N/A",
                            'Underwater Period': "N/A",
                            'Drawdown': f"{((trough / peak) - 1) * 100:.2f}%"
                        })
            peak = value
            peak_date = date
            trough = value
            trough_date = date
        elif value < trough:
            trough = value
            trough_date = date
    if trough < peak:
        drawdowns.append({
            'Start': peak_date.strftime('%b %Y'),
            'End': trough_date.strftime('%b %Y'),
            'Length': f"{(trough_date - peak_date).days // 30} months",
            'Recovery By': "Not Recovered",
            'Recovery Time': "N/A",
            'Underwater Period': "N/A",
            'Drawdown': f"{((trough / peak) - 1) * 100:.2f}%"
        })
    try:
        drawdowns_sorted = sorted(drawdowns, key=lambda x: float(x['Drawdown'].strip('%')), reverse=False)
    except:
        drawdowns_sorted = []
    drawdowns_sorted = drawdowns_sorted[:10]
    for idx, dd in enumerate(drawdowns_sorted, start=1):
        dd['Rank'] = idx
    drawdowns_final = []
    for dd in drawdowns_sorted:
        drawdowns_final.append({
            'Rank': dd['Rank'],
            'Start': dd['Start'],
            'End': dd['End'],
            'Length': dd['Length'],
            'Recovery By': dd['Recovery By'],
            'Recovery Time': dd['Recovery Time'],
            'Underwater Period': dd['Underwater Period'],
            'Drawdown': dd['Drawdown']
        })
    return drawdowns_final

def optimize_portfolio(
    returns, 
    benchmark_returns=None, 
    objectives=['sharpe'], 
    rf=0.02, 
    max_weight=1.0, 
    min_weight=0.0, 
    target_return=None
):
    """
    Optimize a portfolio based on specified objectives.

    Parameters:
    - returns (pd.DataFrame or np.ndarray): Historical returns of assets.
    - benchmark_returns (pd.Series or np.ndarray, optional): Returns of the benchmark index.
    - objectives (list of str): Objectives to optimize. 
        Options include 'sharpe', 'min_variance', 'max_return', 'min_drawdown', 'maximize_alpha', 'minimize_beta'.
    - rf (float): Risk-free rate for alpha calculation.
    - max_weight (float): Maximum weight per asset.
    - min_weight (float): Minimum weight per asset.
    - target_return (float, optional): Target return for the portfolio.

    Returns:
    - np.ndarray: Optimized asset weights or None if optimization fails.
    """
    
    # Define helper functions (ensure these are implemented elsewhere)
    def calculate_beta(portfolio_returns, benchmark_returns):
        covariance = np.cov(portfolio_returns, benchmark_returns)[0, 1]
        variance = np.var(benchmark_returns)
        return covariance / variance if variance != 0 else 0

    def calculate_var(portfolio_returns, confidence_level=0.95):
        if not isinstance(portfolio_returns, np.ndarray):
            portfolio_returns = np.array(portfolio_returns)
        return np.percentile(portfolio_returns, (1 - confidence_level) * 100)

    def calculate_cvar(portfolio_returns, confidence_level=0.95):
        var = calculate_var(portfolio_returns, confidence_level)
        portfolio_returns = np.array(portfolio_returns)
        return portfolio_returns[portfolio_returns <= var].mean()

    # Define individual objective functions
    def sharpe_ratio(weights):
        portfolio_return = np.dot(returns.mean(), weights) * 252
        portfolio_volatility = np.sqrt(np.dot(weights.T, np.dot(returns.cov() * 252, weights)))
        return -portfolio_return / portfolio_volatility if portfolio_volatility > 1e-6 else 0  # Negative for maximization

    def min_variance(weights):
        return np.dot(weights.T, np.dot(returns.cov() * 252, weights))

    def max_return(weights):
        return -np.dot(returns.mean(), weights) * 252  # Negative for maximization

    def min_drawdown(weights):
        # Simplistic drawdown minimization using expected shortfall (CVaR)
        portfolio_returns = returns.dot(weights)
        cvar = calculate_cvar(portfolio_returns, 0.95)
        return cvar  # Minimizing CVaR as a proxy for drawdown

    # Initialize list of objective functions
    objective_functions = []
    
    # Iterate through selected objectives and append corresponding functions
    for obj in objectives:
        if obj == 'sharpe':
            objective_functions.append(sharpe_ratio)
        elif obj == 'min_variance':
            objective_functions.append(min_variance)
        elif obj == 'max_return':
            objective_functions.append(max_return)
        elif obj == 'min_drawdown':
            objective_functions.append(min_drawdown)
        elif obj == 'maximize_alpha':
            if benchmark_returns is None:
                st.error("benchmark_returns must be provided for 'maximize_alpha' objective.")
                return None
            
            benchmark_return = benchmark_returns.mean() * 252  # Assuming daily returns annualized
            def maximize_alpha(weights):
                portfolio_return = np.dot(returns.mean(), weights) * 252
                portfolio_volatility = np.sqrt(np.dot(weights.T, np.dot(returns.cov() * 252, weights)))
                portfolio_returns = returns.dot(weights)
                beta = calculate_beta(portfolio_returns, benchmark_returns)
                alpha = portfolio_return - (rf + beta * (benchmark_return - rf))
                return -alpha  # Negative for maximization

            objective_functions.append(maximize_alpha)
        elif obj == 'minimize_beta':
            if benchmark_returns is None:
                st.error("benchmark_returns must be provided for 'minimize_beta' objective.")
                return None
            
            def minimize_beta(weights):
                portfolio_returns = returns.dot(weights)
                beta = calculate_beta(portfolio_returns, benchmark_returns)
                return beta

            objective_functions.append(minimize_beta)
        else:
            st.error(f"Invalid optimization objective: {obj}")
            return None

    # Composite objective function: weighted sum of individual objectives
    def composite_objective(weights):
        return sum(fn(weights) for fn in objective_functions)

    # Define constraints
    constraints = [
        {'type': 'eq', 'fun': lambda x: np.sum(x) - 1},
    ]

    if target_return is not None:
        def target_return_constraint(x):
            return np.dot(x, returns.mean()) * 252 - target_return
        constraints.append({
            'type': 'ineq',
            'fun': target_return_constraint
        })

    # Define bounds for weights
    bounds = tuple((min_weight, max_weight) for _ in range(returns.shape[1]))
    
    # Initial guess (equally distributed weights)
    initial_guess = np.array([1.0 / returns.shape[1]] * returns.shape[1])

    # Perform optimization
    result = minimize(
        composite_objective, 
        initial_guess, 
        method='SLSQP', 
        bounds=bounds, 
        constraints=constraints
    )
    
    if result.success:
        optimized_weights = result.x
        portfolio_volatility = np.sqrt(np.dot(optimized_weights.T, np.dot(returns.cov() * 252, optimized_weights)))
        
        if portfolio_volatility <= 1e-6:
            st.error("Optimized portfolio has near-zero volatility. Optimization constraints may be too restrictive.")
            return None
        
        return optimized_weights
    else:
        st.error("Optimization failed. Try adjusting your constraints or target return.")
        return None

def monte_carlo_simulation(returns, num_simulations=1000, periods=252, mean_returns=None, cov_matrix=None,
                           mean_reversion=False, mean_reversion_speed=0.1, long_term_mean=None,
                           time_varying_vol=False, vol_change_rate=0.0,
                           stress_shocks=None, stress_period=None,
                           return_distribution='normal'):
    if mean_returns is None:
        mean_returns = returns.mean()
    if cov_matrix is None:
        cov_matrix = returns.cov()

    num_assets = returns.shape[1]
    available_selected = returns.columns.tolist()

    def simulate():
        weights = np.random.random(num_assets)
        weights /= np.sum(weights)
        simulated_prices = [1]  # Start with initial price of 1
        current_mean = mean_returns.copy()
        current_vol = np.sqrt(np.diag(cov_matrix))
        for t in range(periods):
            if mean_reversion:
                current_mean += mean_reversion_speed * (long_term_mean - current_mean)
            if time_varying_vol:
                current_vol += vol_change_rate
            adjusted_cov_matrix = np.outer(current_vol, current_vol) * np.corrcoef(returns.T)
            if return_distribution == 'log-normal':
                simulated_returns = np.random.lognormal(mean=np.log(1 + current_mean), sigma=current_vol) - 1
            else:
                simulated_returns = np.random.multivariate_normal(current_mean, adjusted_cov_matrix)
            
            # Apply stress shocks if applicable
            if stress_shocks and t == stress_period:
                for idx, ticker in enumerate(available_selected):
                    simulated_returns[idx] += stress_shocks.get(ticker, 0)
            
            portfolio_return = np.dot(simulated_returns, weights)
            simulated_prices.append(simulated_prices[-1] * (1 + portfolio_return))
        cumulative_return = simulated_prices[-1] - 1
        return cumulative_return

    portfolio_returns = Parallel(n_jobs=-1)(
        delayed(simulate)() for _ in range(num_simulations)
    )

    return np.array(portfolio_returns)

def plot_efficient_frontier(returns, num_portfolios=1000, rf=0.0):
    try:
        num_assets = returns.shape[1] if returns.ndim > 1 else 1

        def generate_portfolio(returns, rf):
            weights = np.random.random(num_assets)
            weights /= np.sum(weights)
            portfolio_return = np.sum(returns.mean() * weights) * 252
            portfolio_std_dev = np.sqrt(np.dot(weights.T, np.dot(returns.cov() * 252, weights)))
            sharpe_ratio = (portfolio_return - rf) / portfolio_std_dev if portfolio_std_dev > 0 else 0
            return portfolio_std_dev, portfolio_return, sharpe_ratio

        results = Parallel(n_jobs=-1)(
            delayed(generate_portfolio)(returns, rf) for _ in range(num_portfolios)
        )

        # Filter out incomplete results (where any element is NaN)
        valid_results = [res for res in results if all(not np.isnan(x) for x in res)]

        if len(valid_results) == 0:
            st.warning("No valid portfolios to plot on the Efficient Frontier.")
            return

        results = np.array(valid_results).T
        ef_df = pd.DataFrame({
            'Std Dev': results[0],
            'Return': results[1],
            'Sharpe Ratio': results[2]
        })
        fig = px.scatter(
            ef_df,
            x='Std Dev',
            y='Return',
            color='Sharpe Ratio',
            color_continuous_scale='Viridis',
            title='Efficient Frontier',
            labels={
                'Std Dev': 'Annualized Volatility (Std Dev)',
                'Return': 'Annualized Return',
                'Sharpe Ratio': 'Sharpe Ratio'
            },
            hover_data={
                'Sharpe Ratio': ':.2f',
                'Std Dev': ':.2f',
                'Return': ':.2f'
            },
            template='plotly_dark'
        )

        # Add annotation for the maximum Sharpe ratio portfolio
        max_sharpe_idx = ef_df['Sharpe Ratio'].idxmax()
        max_sharpe = ef_df.loc[max_sharpe_idx]
        fig.add_annotation(
            x=max_sharpe['Std Dev'],
            y=max_sharpe['Return'],
            text="Max Sharpe",
            showarrow=True,
            arrowhead=1
        )

        # Check for Optimized Portfolio in Session State
        optimized_weights = st.session_state.backtest_results.get('optimized_weights', None)
        optimized_return = st.session_state.backtest_results.get('optimized_return', None)
        optimized_volatility = st.session_state.backtest_results.get('optimized_volatility', None)
        optimized_sharpe = st.session_state.backtest_results.get('optimized_sharpe', None)

        if optimized_weights is not None and optimized_return is not None and optimized_volatility is not None and optimized_sharpe is not None:
            if optimized_volatility > 1e-6 and np.isfinite(optimized_sharpe):
                # Add optimized portfolio to the existing Efficient Frontier plot
                fig.add_trace(go.Scatter(
                    x=[optimized_volatility],
                    y=[optimized_return],
                    mode='markers+text',
                    marker=dict(color='red', size=14, symbol='star'),
                    name='Optimized Portfolio',
                    text=["Optimized"],
                    textposition="top center",
                    hoverinfo='text'
                ))
                fig.add_annotation(
                    x=optimized_volatility,
                    y=optimized_return,
                    text="Optimized",
                    showarrow=True,
                    arrowhead=2,
                    ax=0,
                    ay=-40
                )

        st.plotly_chart(fig, use_container_width=True, config={
            'responsive': True,
            'scrollZoom': True,
            'displayModeBar': True
        })
    except Exception as e:
        st.error(f"Error plotting efficient frontier: {e}")

def generate_portfolio(returns, rf):
    try:
        weights = np.random.random(returns.shape[1])
        weights /= np.sum(weights)
        portfolio_return = np.dot(weights, returns.mean()) * 252
        portfolio_volatility = np.sqrt(np.dot(weights.T, np.dot(returns.cov() * 252, weights)))
        sharpe_ratio = (portfolio_return - rf) / portfolio_volatility if portfolio_volatility > 0 else 0
        return portfolio_volatility, portfolio_return, sharpe_ratio
    except Exception as e:
        st.error(f"Error generating portfolio: {e}")
        return np.nan, np.nan, np.nan  # Return NaNs to indicate failure

def plot_allocation_pie(weights, assets, title='Portfolio Allocation', hover_info=None):
    allocation_df = pd.DataFrame({
        'Asset': assets,
        'Weight': weights
    })
    
    # Configure hover data based on hover_info parameter
    if hover_info == "percent+name":
        hover_data = ['Weight']  # Changed from dict to list
    elif hover_info == "name":
        hover_data = []  # No additional hover data
    elif hover_info == "percent":
        hover_data = ['Weight']  # Changed from dict to list
    else:
        hover_data = []  # No additional hover data
    
    fig = px.pie(
        allocation_df,
        names='Asset',
        values='Weight',
        title=title,
        color='Asset',
        color_discrete_sequence=px.colors.qualitative.Set3,
        hover_data=hover_data
    )
    st.plotly_chart(fig, use_container_width=True)

def plot_rolling_cagr(cum_returns, window=252):
    try:
        rolling_years = window / 252  # Assuming daily data
        rolling_cagr = (cum_returns / cum_returns.shift(window)) ** (1/rolling_years) - 1
        fig = px.line(
            rolling_cagr.dropna(),
            title=f'Rolling {int(rolling_years)}-Year CAGR',
            labels={'x': 'Date', 'y': 'Rolling CAGR'},
            template='plotly_dark'
        )
        st.plotly_chart(fig, use_container_width=True)
    except Exception as e:
        st.error(f"Error plotting rolling CAGR: {e}")

def plot_correlation_heatmap(returns):
    try:
        if isinstance(returns, pd.DataFrame) and returns.shape[1] >= 2:
            valid_columns = [col for col in returns.columns if returns[col].nunique() > 1 and returns[col].notna().sum() >= 10]
            if len(valid_columns) < 2:
                st.warning("Correlation heatmap requires at least two assets with variability and sufficient data.")
                return
            corr_matrix = returns[valid_columns].corr()
            fig = px.imshow(
                corr_matrix,
                title='Asset Correlation Heatmap',
                labels={'x': 'Asset', 'y': 'Asset', 'color': 'Correlation'},
                color_continuous_scale='Portland',  # Updated color scale
                zmin=-1,
                zmax=1,
                text_auto=True,
                aspect="auto",
                template='plotly_dark'               # Dark theme for better contrast
            )
            fig.update_xaxes(side="top")  # Move x-axis labels to the top for better readability
            st.plotly_chart(fig, use_container_width=True)
        elif isinstance(returns, pd.Series) and returns.nunique() > 1:
            st.warning("Correlation heatmap requires at least two assets with variability.")
        else:
            st.warning("Correlation heatmap requires at least two assets with variability.")
    except Exception as e:
        st.error(f"Error plotting correlation heatmap: {e}")

def plot_rolling_metrics(returns, windows=[30, 90, 180, 252]):
    metrics = {}
    for window in windows:
        rolling_return = returns.rolling(window).mean() * 252
        rolling_vol = returns.rolling(window).std() * np.sqrt(252)
        rolling_sharpe = (rolling_return - 0.02) / rolling_vol
        metrics[f'Rolling {window}-Day Volatility'] = rolling_vol
        metrics[f'Rolling {window}-Day Sharpe Ratio'] = rolling_sharpe
    fig = go.Figure()
    for metric_name, metric_series in metrics.items():
        fig.add_trace(go.Scatter(x=metric_series.index, y=metric_series, mode='lines', name=metric_name))
    fig.update_layout(
        title='Rolling Metrics Over User-Defined Periods',
        xaxis_title='Date',
        yaxis_title='Value',
        hovermode='x unified'
    )
    st.plotly_chart(fig, use_container_width=True)

def plot_risk_return_attribution(returns, weights):
    try:
        if isinstance(returns, pd.Series):
            asset_returns = returns.mean() * 252
            asset_volatility = returns.std() * np.sqrt(252)
            asset_contribution = weights[0] * asset_volatility if len(weights) > 0 else 0
            hover_names = [returns.name] if returns.name else ['Asset']
            x = [asset_volatility]
            y = [asset_returns]
            size = [asset_contribution]
            color = [weights[0]]
        else:
            asset_returns = returns.mean() * 252
            asset_volatility = returns.std() * np.sqrt(252)
            asset_contribution = weights * asset_volatility
            hover_names = returns.columns.tolist()
            x = asset_volatility.values
            y = asset_returns.values
            size = asset_contribution
            color = weights

        fig = px.scatter(
            x=x,
            y=y,
            size=size,
            color=color,
            hover_name=hover_names,
            title='Risk-Return Attribution',
            labels={'x': 'Annualized Volatility', 'y': 'Annualized Return', 'color': 'Weight (%)'},
            size_max=60,
            color_continuous_scale='Viridis'
        )
        fig.update_layout(template='plotly_dark')
        st.plotly_chart(fig, use_container_width=True)
    except Exception as e:
        st.error(f"Error plotting risk-return attribution: {e}")

def plot_var_cvar_distribution(portfolio_returns, var, cvar):
    fig = px.histogram(portfolio_returns, nbins=50, title='Returns Distribution with VaR and CVaR',
                       labels={'value': 'Returns', 'count': 'Frequency'})
    fig.add_vline(x=var, line_dash="dash", line_color="red", annotation_text=f"VaR: {var:.2f}", annotation_position="top left")
    fig.add_vline(x=cvar, line_dash="dash", line_color="blue", annotation_text=f"CVaR: {cvar:.2f}", annotation_position="top left")
    st.plotly_chart(fig, use_container_width=True)

def plot_var_cvar_over_time(portfolio_returns, var, cvar):
    cumulative_returns = (1 + portfolio_returns).cumprod()
    fig = px.line(cumulative_returns, title='Cumulative Returns with VaR and CVaR Over Time', labels={'value': 'Cumulative Returns', 'index': 'Date'})
    fig.add_hline(y=var, line_dash="dash", line_color="red", annotation_text=f"VaR: {var:.2f}", annotation_position="bottom right")
    fig.add_hline(y=cvar, line_dash="dash", line_color="blue", annotation_text=f"CVaR: {cvar:.2f}", annotation_position="bottom right")
    st.plotly_chart(fig, use_container_width=True)

def calculate_var(returns, confidence_level=0.95):
    return np.percentile(returns, 100 * (1 - confidence_level))

def calculate_cvar(returns, confidence_level=0.95):
    var = calculate_var(returns, confidence_level)
    return returns[returns <= var].mean()

# ----------------------------
# Caching Functions
# ----------------------------
@st.cache_data(show_spinner=False)
def calculate_portfolio_metrics(returns, cum_returns, rf, benchmark_returns=None):
    metrics = calculate_simple_metrics(returns, cum_returns)
    
    if benchmark_returns is not None and not benchmark_returns.empty:
        correlation = returns.corr(benchmark_returns)
        beta = calculate_beta(returns, benchmark_returns)
        metrics['Benchmark Correlation'] = f"{correlation:.2f}" if not np.isnan(correlation) else np.nan
        metrics['Beta'] = f"{beta:.2f}" if not np.isnan(beta) else np.nan

        alpha = calculate_alpha(returns, benchmark_returns, rf)
        metrics['Alpha (annualized)'] = f"{alpha * 100:.2f}%" if not np.isnan(alpha) else np.nan

        r2 = calculate_r_squared(returns, benchmark_returns)
        metrics['R2'] = f"{r2 * 100:.2f}%" if not np.isnan(r2) else np.nan

        treynor = calculate_treynor_ratio(returns, benchmark_returns, rf)
        metrics['Treynor Ratio'] = f"{treynor:.2f}" if not np.isnan(treynor) else np.nan

        calmar = calculate_calmar_ratio(returns, cum_returns)
        metrics['Calmar Ratio'] = f"{calmar:.2f}" if not np.isnan(calmar) else np.nan

        m2 = calculate_modigliani_miller(returns, benchmark_returns, rf)
        metrics['Modigliani–Modigliani Measure'] = f"{m2 * 100:.2f}%" if not np.isnan(m2) else np.nan

        info_ratio = calculate_information_ratio(returns, benchmark_returns)
        metrics['Information Ratio'] = f"{info_ratio:.2f}" if not np.isnan(info_ratio) else np.nan

        tracking_error = calculate_tracking_error(returns, benchmark_returns)
        metrics['Tracking Error'] = f"{tracking_error * 100:.2f}%" if not np.isnan(tracking_error) else np.nan

        active_return = calculate_active_return(returns, benchmark_returns)
        metrics['Active Return'] = f"{active_return:.2f}%" if not np.isnan(active_return) else np.nan

        upside_capture = calculate_capture_ratio(returns, benchmark_returns, upside=True)
        downside_capture = calculate_capture_ratio(returns, benchmark_returns, upside=False)
        metrics['Upside Capture Ratio'] = f"{upside_capture:.2f}%" if not np.isnan(upside_capture) else np.nan
        metrics['Downside Capture Ratio'] = f"{downside_capture:.2f}%" if not np.isnan(downside_capture) else np.nan
    else:
        metrics.update({
            'Benchmark Correlation': np.nan,
            'Beta': np.nan,
            'Alpha (annualized)': np.nan,
            'R2': np.nan,
            'Treynor Ratio': np.nan,
            'Calmar Ratio': np.nan,
            'Modigliani–Modigliani Measure': np.nan,
            'Information Ratio': np.nan,
            'Tracking Error': np.nan,
            'Active Return': np.nan,
            'Upside Capture Ratio': np.nan,
            'Downside Capture Ratio': np.nan
        })
    
    return metrics

@st.cache_data(show_spinner=False)
def get_tickers():
    try:
        # Fetch S&P 500 companies with requests, bypassing SSL verification
        sp500_url = 'https://en.wikipedia.org/wiki/List_of_S%26P_500_companies'
        sp500_response = requests.get(sp500_url, verify=False)
        sp500_table = pd.read_html(sp500_response.text)[0]
        sp500 = sp500_table[['Symbol', 'Security']].rename(columns={'Symbol': 'Ticker', 'Security': 'Company Name'})
        sp500['Ticker'] = sp500['Ticker'].str.replace('.', '-', regex=False)

        # Fetch NASDAQ-100 companies with requests, bypassing SSL verification
        nasdaq100_url = 'https://en.wikipedia.org/wiki/NASDAQ-100'
        nasdaq100_response = requests.get(nasdaq100_url, verify=False)
        nasdaq100_tables = pd.read_html(nasdaq100_response.text)
        
        nasdaq100 = pd.DataFrame()
        for table in nasdaq100_tables:
            if 'Ticker' in table.columns and 'Company' in table.columns:
                nasdaq100 = table[['Ticker', 'Company']].rename(columns={'Ticker': 'Ticker', 'Company': 'Company Name'})
                nasdaq100['Ticker'] = nasdaq100['Ticker'].str.replace('.', '-', regex=False)
                break

        combined = pd.concat([sp500, nasdaq100], ignore_index=True)
        combined = combined.drop_duplicates(subset=['Ticker'])
        return combined.sort_values('Ticker').reset_index(drop=True)
    except Exception as e:
        st.error(f"Error fetching tickers: {e}")
        return pd.DataFrame(columns=['Ticker', 'Company Name'])

@st.cache_data(show_spinner=False)
def download_data(tickers, start, end, retries=3, backoff_factor=0.3):
    try:
        # Configure retry strategy for requests (used by yfinance internally)
        session = requests.Session()
        retry = Retry(
            total=retries,
            read=retries,
            connect=retries,
            backoff_factor=backoff_factor,
            status_forcelist=(500, 502, 504),
        )
        adapter = HTTPAdapter(max_retries=retry)
        session.mount('http://', adapter)
        session.mount('https://', adapter)
        
        # Attempt to download data
        data = yf.download(tickers, start=start, end=end, progress=False, session=session)['Adj Close']
        
        # Handle potential empty data
        if isinstance(data, pd.Series):
            data = data.to_frame()
        if data.empty:
            st.warning("No price data available for the selected portfolio. Please check the ticker symbols and date range.")
            return pd.DataFrame()
        if data.index.tz is not None:
            data.index = data.index.tz_localize(None)
        
        # Fill missing data
        data = data.fillna(method='ffill').fillna(method='bfill')
        if data.isnull().values.any():
            st.warning("Data contains missing values after filling. Some calculations may be affected.")
        return data
    except Exception as e:
        st.error(f"Error downloading data: {e}")
        return pd.DataFrame()

# ----------------------------
# Initialize Session State
# ----------------------------
if 'portfolios' not in st.session_state:
    st.session_state.portfolios = []
if 'backtest_results' not in st.session_state:
    st.session_state.backtest_results = {}
if 'step' not in st.session_state:
    st.session_state.step = "Configure Portfolio"
if 'edit_portfolio' not in st.session_state:
    st.session_state.edit_portfolio = None
if 'default_config' not in st.session_state:
    st.session_state.default_config = {
        'rf_rate': 0.02,
        'broker_fee': 0.0,
        'benchmark_symbol': '^GSPC'
    }

# ----------------------------
# Streamlit Layout
# ----------------------------
st.set_page_config(page_title="🎯 Portfolio Optimizer", layout="wide")
st.title("🎯 Portfolio Optimizer")

# Use a placeholder that hides after loading
loading_placeholder = st.empty()
loading_placeholder.info("🎯 Portfolio Optimizer is loading, please be patient...")

# After loading is complete, clear the placeholder
loading_placeholder.empty()

st.sidebar.header("📂 Navigation")
step = st.sidebar.radio("Navigate to", [
    "Configure Portfolio",
    "Run Backtest",
    "Optimize Portfolio",
    "Monte Carlo Simulations",
    "Risk Analysis"
], index=["Configure Portfolio", "Run Backtest", "Optimize Portfolio", "Monte Carlo Simulations", "Risk Analysis"].index(st.session_state.get('step', "Configure Portfolio")), key="sidebar_radio")

tickers = get_tickers()

if step == "Configure Portfolio":
    col1, col2 = st.columns([1,1])
    with col1:
        st.subheader("📁 Existing Portfolios")
        if st.session_state.portfolios:
            portfolio_names = [p['name'] for p in st.session_state.portfolios]
            selected_portfolio = st.selectbox("Select Portfolio to Edit/Delete/Duplicate", portfolio_names)
            if selected_portfolio:
                portfolio = next(p for p in st.session_state.portfolios if p['name'] == selected_portfolio)
                col_del, col_edit, col_dup = st.columns(3)
                with col_del:
                    if st.button("🗑️ Delete Portfolio"):
                        st.session_state.portfolios = [p for p in st.session_state.portfolios if p['name'] != selected_portfolio]
                        st.success("Portfolio deleted.")
                        # Reset edit_portfolio if it was the deleted portfolio
                        if st.session_state.edit_portfolio == selected_portfolio:
                            st.session_state.edit_portfolio = None
                with col_edit:
                    if st.button("✏️ Edit Portfolio"):
                        st.session_state.edit_portfolio = selected_portfolio
                with col_dup:
                    if st.button("📄 Duplicate Portfolio"):
                        new_name = st.text_input("New Portfolio Name", f"{selected_portfolio}_Copy")
                        if st.button("Confirm Duplicate"):
                            if not new_name:
                                st.error("Please provide a new portfolio name.")
                            elif new_name in [p['name'] for p in st.session_state.portfolios]:
                                st.error("Portfolio name already exists.")
                            else:
                                duplicated = portfolio.copy()
                                duplicated['name'] = new_name
                                st.session_state.portfolios.append(duplicated)
                                st.success("Portfolio duplicated successfully.")
        else:
            st.write("No portfolios saved.")
        # Edit Portfolio Section
        if st.session_state.edit_portfolio:
            st.markdown("---")
            st.subheader("✏️ Edit Portfolio")
            try:
                portfolio_to_edit = next(p for p in st.session_state.portfolios if p['name'] == st.session_state.edit_portfolio)
            except StopIteration:
                st.error("The portfolio you are trying to edit no longer exists.")
                st.session_state.edit_portfolio = None
            else:
                with st.form("edit_portfolio_form"):
                    name_edit = st.text_input("Portfolio Name", portfolio_to_edit['name'], help="Enter a unique name for your portfolio.")
                    start_date_edit = st.date_input(
                        "Start Date:",
                        portfolio_to_edit['start_date'].date(),
                        min_value=datetime(1900, 1, 1),
                        help="Choose the start date for backtesting."
                    )
                    end_date_edit = st.date_input(
                        "End Date:",
                        portfolio_to_edit['end_date'].date(),
                        min_value=datetime(1900, 1, 1),
                        help="Choose the end date for backtesting."
                    )
                    rf_rate_edit = st.number_input(
                        "Risk-Free Rate (%)",
                        0.0,
                        10.0,
                        portfolio_to_edit['rf_rate'] * 100,
                        help="Enter the risk-free rate as a percentage."
                    ) / 100
                    broker_fee_edit = st.number_input(
                        "Broker Fee (%)",
                        0.0,
                        10.0,
                        portfolio_to_edit['broker_fee'] * 100,
                        step=0.000001,
                        format="%.6f",
                        help="Enter the broker fee as a percentage per transaction."
                    ) / 100
                    benchmark_symbol_edit = st.text_input(
                        "Benchmark Symbol (e.g., ^GSPC)",
                        portfolio_to_edit['benchmark_symbol'],
                        help="Enter the ticker symbol for your benchmark index."
                    )
                    rebalance_freq_edit = st.selectbox(
                        "Rebalance Frequency",
                        ["Daily", "Weekly", "Monthly", "Quarterly", "Yearly"],
                        index=["Daily", "Weekly", "Monthly", "Quarterly", "Yearly"].index(
                            [k for k, v in frequency_mapping.items() if v == portfolio_to_edit['rebalance_freq']][0]
                        ),
                        help="Choose how often to rebalance the portfolio."
                    )
                    st.markdown("### 🛠️ Set Allocations")
                    # Set default selections as formatted strings based on existing tickers
                    selected_default = [format_asset_option(ticker, get_company_name(tickers, ticker)) for ticker in portfolio_to_edit['selected']]
                    ticker_df = get_tickers()
                    asset_options = ticker_df.apply(lambda row: format_asset_option(row['Ticker'], row['Company Name']), axis=1).tolist()
                    selected_edit = st.multiselect(
                        "Select Assets:",
                        options=asset_options,
                        default=selected_default,  # Use formatted strings for default
                        help="Choose the assets you want to include in your portfolio."
                    )
                    selected_tickers_edit = [option.split(' - ')[0] for option in selected_edit if ' - ' in option]
                    allocations_edit = []
                    if selected_tickers_edit:
                        for ticker in selected_tickers_edit:
                            # Retrieve the existing allocation for the ticker
                            try:
                                alloc = portfolio_to_edit['allocations'][portfolio_to_edit['selected'].index(ticker)]
                            except ValueError:
                                alloc = 0.0  # Default to 0.0% if ticker not found

                            # Set the number input value to the existing allocation
                            alloc_input = st.number_input(
                                f"{ticker} Allocation (%)",
                                min_value=0.0,
                                max_value=100.0,
                                value=alloc,  # Use existing allocation here
                                step=0.000001,
                                format="%.6f",
                                key=f"alloc_edit_{ticker}",
                                help=f"Set the allocation percentage for {ticker}. Must sum to 100% across all selected assets."
                            )
                            allocations_edit.append(alloc_input)
                        current_total_edit = sum(allocations_edit)
                        st.markdown(f"**Total Allocation:** {current_total_edit:.6f}%")
                        if not np.isclose(current_total_edit, 100.0, atol=1e-4):
                            st.warning(f"Allocations must sum to 100%. Currently sum to {current_total_edit:.6f}%. Please adjust the allocations.")
                            if current_total_edit > 0:
                                allocations_edit = [alloc / current_total_edit * 100 for alloc in allocations_edit]
                            else:
                                st.error("Total allocation is zero. Please set allocations for your assets.")
                    submitted_edit = st.form_submit_button("Save Changes")
                    if submitted_edit:
                        if not name_edit:
                            st.error("Please provide a portfolio name.")
                        elif name_edit != portfolio_to_edit['name'] and name_edit in [p['name'] for p in st.session_state.portfolios]:
                            st.error("Portfolio name already exists. Please choose a unique name.")
                        elif not selected_edit:
                            st.error("Please select at least one asset.")
                        elif not np.isclose(sum(allocations_edit), 100.0, atol=1e-4):
                            st.warning(f"Allocations must sum to 100%. Currently sum to {sum(allocations_edit):.6f}%. Adjusting allocations proportionally.")
                            allocations_edit = [alloc / current_total_edit * 100 for alloc in allocations_edit]
                        portfolio_to_edit.update({
                            'name': name_edit,
                            'start_date': pd.to_datetime(start_date_edit),
                            'end_date': pd.to_datetime(end_date_edit),
                            'rf_rate': rf_rate_edit,
                            'broker_fee': broker_fee_edit,
                            'benchmark_symbol': benchmark_symbol_edit,
                            'rebalance_freq': frequency_mapping.get(rebalance_freq_edit, 'M'),
                            'selected': selected_tickers_edit,
                            'allocations': allocations_edit
                        })
                        st.success(f"Portfolio '{name_edit}' updated successfully!")
                        st.markdown("**Updated Allocations:**")
                        allocations_summary = {ticker: f"{alloc:.6f}%" for ticker, alloc in zip(selected_tickers_edit, allocations_edit)}
                        st.json(allocations_summary)
                        st.session_state.edit_portfolio = None
        st.markdown("---")
        st.subheader("💡 Example Portfolios")
        example_portfolios = [
            {
                'name': "Retirement Portfolio",
                'start_date': pd.to_datetime(datetime.today() - timedelta(days=365*10)),
                'end_date': pd.to_datetime(datetime.today()),
                'rf_rate': 0.02,
                'broker_fee': 0.0,
                'benchmark_symbol': '^GSPC',
                'rebalance_freq': 'M',
                'selected': ['AAPL', 'MSFT', 'GOOGL', 'JPM', 'XOM'],
                'allocations': [20.0, 20.0, 20.0, 20.0, 20.0]
            },
            {
                'name': "Tech-Heavy Portfolio",
                'start_date': pd.to_datetime(datetime.today() - timedelta(days=365*5)),
                'end_date': pd.to_datetime(datetime.today()),
                'rf_rate': 0.02,
                'broker_fee': 0.0,
                'benchmark_symbol': '^IXIC',
                'rebalance_freq': 'M',
                'selected': ['AAPL', 'MSFT', 'GOOGL', 'AMZN', 'META'],
                'allocations': [25.0, 25.0, 20.0, 15.0, 15.0]
            },
            {
                'name': "Balanced Risk-Return Portfolio",
                'start_date': pd.to_datetime(datetime.today() - timedelta(days=365*7)),
                'end_date': pd.to_datetime(datetime.today()),
                'rf_rate': 0.02,
                'broker_fee': 0.0,
                'benchmark_symbol': '^GSPC',
                'rebalance_freq': 'M',
                'selected': ['AAPL', 'JNJ', 'V', 'PG', 'XOM'],
                'allocations': [20.0, 20.0, 20.0, 20.0, 20.0]
            },
            {
                'name': "Income-Focused Portfolio",
                'start_date': pd.to_datetime(datetime.today() - timedelta(days=365*7)),
                'end_date': pd.to_datetime(datetime.today()),
                'rf_rate': 0.02,
                'broker_fee': 0.0,
                'benchmark_symbol': '^DJI',
                'rebalance_freq': 'Q',
                'selected': ['PG', 'KO', 'JNJ', 'T', 'PFE'],
                'allocations': [20.0, 20.0, 20.0, 20.0, 20.0]
            },
            {
                'name': "Growth-Oriented Portfolio",
                'start_date': pd.to_datetime(datetime.today() - timedelta(days=365*3)),
                'end_date': pd.to_datetime(datetime.today()),
                'rf_rate': 0.02,
                'broker_fee': 0.0,
                'benchmark_symbol': '^IXIC',
                'rebalance_freq': 'M',
                'selected': ['TSLA', 'NVDA', 'AMD', 'META', 'NFLX'],
                'allocations': [30.0, 25.0, 20.0, 15.0, 10.0]
            },
            {
                'name': "Value Portfolio",
                'start_date': pd.to_datetime(datetime.today() - timedelta(days=365*10)),
                'end_date': pd.to_datetime(datetime.today()),
                'rf_rate': 0.02,
                'broker_fee': 0.0,
                'benchmark_symbol': '^DJI',
                'rebalance_freq': 'Q',
                'selected': ['KO', 'PFE', 'XOM', 'WMT', 'CVX'],
                'allocations': [20.0, 20.0, 20.0, 20.0, 20.0]
            },
            {
                'name': "Conservative Portfolio",
                'start_date': pd.to_datetime(datetime.today() - timedelta(days=365*5)),
                'end_date': pd.to_datetime(datetime.today()),
                'rf_rate': 0.02,
                'broker_fee': 0.0,
                'benchmark_symbol': '^GSPC',
                'rebalance_freq': 'Q',
                'selected': ['JNJ', 'PG', 'KO', 'PEP', 'WMT'],
                'allocations': [20.0, 20.0, 20.0, 20.0, 20.0]
            },
            {
                'name': "Dividend-Focused Portfolio",
                'start_date': pd.to_datetime(datetime.today() - timedelta(days=365*7)),
                'end_date': pd.to_datetime(datetime.today()),
                'rf_rate': 0.02,
                'broker_fee': 0.0,
                'benchmark_symbol': 'DVY',  # Dividend Index
                'rebalance_freq': 'Q',
                'selected': ['T', 'VZ', 'PFE', 'KO', 'IBM'],
                'allocations': [20.0, 20.0, 20.0, 20.0, 20.0]
            },
            {
                'name': "Global Diversification Portfolio",
                'start_date': pd.to_datetime(datetime.today() - timedelta(days=365*5)),
                'end_date': pd.to_datetime(datetime.today()),
                'rf_rate': 0.02,
                'broker_fee': 0.0,
                'benchmark_symbol': 'ACWI',  # All Country World Index
                'rebalance_freq': 'M',
                'selected': ['AAPL', 'TSM', 'BABA', 'SAP', 'UL'],
                'allocations': [20.0, 20.0, 20.0, 20.0, 20.0]
            },
            {
                'name': "High-Risk Growth Portfolio",
                'start_date': pd.to_datetime(datetime.today() - timedelta(days=365*3)),
                'end_date': pd.to_datetime(datetime.today()),
                'rf_rate': 0.02,
                'broker_fee': 0.0,
                'benchmark_symbol': '^IXIC',
                'rebalance_freq': 'M',
                'selected': ['TSLA', 'ARKK', 'NVDA', 'SHOP', 'CRWD'],
                'allocations': [30.0, 20.0, 20.0, 15.0, 15.0]
            },
            {
                'name': "Emerging Markets Portfolio",
                'start_date': pd.to_datetime(datetime.today() - timedelta(days=365*5)),
                'end_date': pd.to_datetime(datetime.today()),
                'rf_rate': 0.02,
                'broker_fee': 0.0,
                'benchmark_symbol': 'EEM',  # Emerging Markets Index
                'rebalance_freq': 'M',
                'selected': ['BABA', 'TSM', 'PDD', 'INFY', 'VALE'],
                'allocations': [25.0, 25.0, 20.0, 15.0, 15.0]
            }
        ]


        for example in example_portfolios:
            with st.expander(example['name']):
                # Display portfolio details in a table
                df_example = pd.DataFrame({
                    'Attribute': ['Start Date', 'End Date', 'Risk-Free Rate (%)', 'Broker Fee (%)', 'Benchmark Symbol', 'Rebalance Frequency', 'Selected Assets', 'Allocations (%)'],
                    'Value': [
                        example['start_date'].strftime('%Y-%m-%d'),
                        example['end_date'].strftime('%Y-%m-%d'),
                        f"{example['rf_rate']*100:.2f}",
                        f"{example['broker_fee']*100:.6f}",
                        example['benchmark_symbol'],
                        example['rebalance_freq'],
                        ", ".join(example['selected']),
                        ", ".join([f"{alloc:.2f}" for alloc in example['allocations']])
                    ]
                })
                st.table(df_example)
                if st.button(f"Load {example['name']}", key=f"load_{example['name']}"):
                    if example['name'] in [p['name'] for p in st.session_state.portfolios]:
                        st.warning(f"Portfolio '{example['name']}' already exists.")
                    else:
                        st.session_state.portfolios.append(example.copy())
                        st.success(f"Portfolio '{example['name']}' loaded successfully!")

    with col2:
        st.subheader("➕ Add New Portfolio")
        with st.form("add_portfolio_form"):
            st.markdown("### 🔧 Configure Details")
            name = st.text_input("Portfolio Name", "", help="Enter a unique name for your portfolio.")
            start_date = st.date_input(
                "Start Date:",
                datetime.today() - timedelta(days=365 * 15),
                min_value=datetime(1900, 1, 1),
                help="Choose the start date for backtesting."
            )
            end_date = st.date_input(
                "End Date:",
                datetime.today(),
                min_value=datetime(1900, 1, 1),
                help="Choose the end date for backtesting."
            )
            rf_rate = st.number_input(
                "Risk-Free Rate (%)",
                0.0,
                10.0,
                st.session_state.default_config['rf_rate'] * 100,
                help="Enter the risk-free rate as a percentage."
            ) / 100
            broker_fee = st.number_input(
                "Broker Fee (%)",
                0.0,
                10.0,
                st.session_state.default_config['broker_fee'] * 100,
                step=0.000001,
                format="%.6f",
                help="Enter the broker fee as a percentage per transaction."
            ) / 100
            benchmark_symbol = st.text_input(
                "Benchmark Symbol (e.g., ^GSPC)",
                st.session_state.default_config['benchmark_symbol'],
                help="Enter the ticker symbol for your benchmark index."
            )
            rebalance_freq = st.selectbox(
                "Rebalance Frequency",
                ["Daily", "Weekly", "Monthly", "Quarterly", "Yearly"],
                index=2,
                help="Choose how often to rebalance the portfolio."
            )
            st.markdown("### 🛠️ Set Allocations")
            ticker_df = get_tickers()
            asset_options = ticker_df.apply(lambda row: format_asset_option(row['Ticker'], row['Company Name']), axis=1).tolist()
            selected = st.multiselect(
                "Select Assets:",
                options=asset_options,
                help="Choose the assets you want to include in your portfolio."
            )
            # After selecting assets, extract tickers
            selected_tickers = [option.split(' - ')[0] for option in selected]
            allocations = []
            if selected_tickers:
                for ticker in selected_tickers:
                    alloc = st.number_input(
                        f"{ticker} Allocation (%)",
                        min_value=0.0,
                        max_value=100.0,
                        value=0.0,
                        step=0.000001,
                        format="%.6f",
                        key=f"allocations_new_{ticker}",
                        help=f"Set the allocation percentage for {ticker}. The sum of all allocations must equal 100%."
                    )
                    allocations.append(alloc)
                current_total = sum(allocations)
                st.markdown(f"**Total Allocation:** {current_total:.6f}%")
                if not np.isclose(current_total, 100.0, atol=1e-4):
                    st.warning(f"Allocations must sum to 100%. Currently sum to {current_total:.6f}%. Please adjust the allocations.")
            submitted = st.form_submit_button("Add Portfolio")
            if submitted:
                if not name:
                    st.error("Please provide a portfolio name.")
                elif name in [p['name'] for p in st.session_state.portfolios]:
                    st.error("Portfolio name already exists. Please choose a unique name.")
                elif not selected:
                    st.error("Please select at least one asset.")
                elif not np.isclose(sum(allocations), 100.0, atol=1e-4):
                    st.error(f"Allocations must sum to 100%. Currently sum to {sum(allocations):.6f}%.")
                else:
                    new_portfolio = {
                        'name': name,
                        'start_date': pd.to_datetime(start_date),
                        'end_date': pd.to_datetime(end_date),
                        'rf_rate': rf_rate,
                        'broker_fee': broker_fee,
                        'benchmark_symbol': benchmark_symbol,
                        'rebalance_freq': frequency_mapping.get(rebalance_freq, 'M'),
                        'selected': selected_tickers,
                        'allocations': allocations
                    }
                    st.session_state.portfolios.append(new_portfolio)
                    st.success(f"Portfolio '{name}' added successfully!")
                    st.markdown("**Allocations:**")
                    allocations_summary = {ticker: f"{alloc}%" for ticker, alloc in zip(selected_tickers, allocations)}
                    st.json(allocations_summary)

    # Debugging Information
    with st.expander("🔍 Debug Information"):
        st.write("Current Portfolios in Session State:")
        st.write(st.session_state.portfolios)

elif step == "Run Backtest":
    st.title("📊 Run Backtest")
    if st.session_state.portfolios:
        with st.form("compare_form"):
            st.subheader("🔍 Select Portfolios to Compare")
            comparison_type = st.selectbox(
                "Comparison Type",
                ["Portfolio vs Benchmark", "Portfolio vs Portfolio"]
            )
            portfolio_names = [p['name'] for p in st.session_state.portfolios]
            if comparison_type == "Portfolio vs Benchmark":
                portfolio1 = st.selectbox("Select Portfolio", portfolio_names)
            else:
                portfolio1 = st.selectbox("Select First Portfolio", portfolio_names, key="p1")
                portfolio2 = st.selectbox("Select Second Portfolio", portfolio_names, key="p2")
            
            st.subheader("⚙️ Visualizations Settings")
            # Moved the selection inputs here
            selected_time_frames = st.multiselect(
                "Select Time Frames for CAGR",
                options=['Weekly', 'Monthly', 'Quarterly', 'Annually'],
                default=['Annually'],
                key="selected_time_frames",
                help="Choose one or more time frames to view CAGR over different horizons."
            )
            selected_rolling_periods = st.multiselect(
                "Select Rolling Periods (Days)",
                options=[30, 90, 180, 252],
                default=[252],
                key="selected_rolling_periods",
                help="Choose one or more periods to view rolling metrics."
            )
            
            run = st.form_submit_button("Run Backtest")

        if run:
            with st.spinner("Running backtest..."):
                if comparison_type == "Portfolio vs Benchmark":
                    portfolio = next(p for p in st.session_state.portfolios if p['name'] == portfolio1)

                    price_data = download_data(
                        portfolio['selected'],
                        portfolio['start_date'],
                        portfolio['end_date']
                    )
                    if price_data.empty:
                        st.error("No price data available for the selected portfolio.")
                    else:
                        available_selected = [ticker for ticker in portfolio['selected'] if ticker in price_data.columns]
                        missing_selected = list(set(portfolio['selected']) - set(available_selected))
                        if missing_selected:
                            st.warning(f"Excluded tickers with no data: {', '.join(missing_selected)}")
                        if not available_selected:
                            st.error(
                                "No selected tickers have available data for the chosen date range. "
                                "Please ensure that the ticker symbols are correct and data is available for the specified period."
                            )
                        else:
                            allocations = [portfolio.get('allocations', [0.0]*len(portfolio.get('selected', [])))[i]
                                          for i, ticker in enumerate(portfolio.get('selected', [])) if ticker in available_selected]
                            if not np.isclose(sum(allocations), 100.0, atol=1e-4):
                                st.warning("Allocations do not sum to 100%. Adjusting allocations proportionally.")
                                allocations = [a / sum(allocations) * 100 for a in allocations]
                            weights = np.array(allocations) / 100
                            returns, cum_returns = backtest(weights, price_data[available_selected],
                                                            portfolio['rebalance_freq'], portfolio['broker_fee'], debug=False)
                            if cum_returns.empty:
                                st.error("Cumulative returns are empty. Check the data and allocations.")
                            else:
                                benchmark_symbol = portfolio['benchmark_symbol']
                                benchmark_data = download_data([benchmark_symbol], portfolio['start_date'], portfolio['end_date'])

                                if benchmark_symbol not in benchmark_data.columns:
                                    st.error(f"The selected benchmark symbol '{benchmark_symbol}' does not have available data for the chosen period.")
                                    st.stop()
                                else:
                                    st.success(f"Benchmark '{benchmark_symbol}' data successfully downloaded and will be used for comparison.")
                                    if not benchmark_data.empty:
                                        benchmark_returns = benchmark_data[benchmark_symbol].pct_change().dropna()
                                        benchmark_returns = benchmark_returns.reindex(returns.index, method='ffill').dropna()
                                        common_index = returns.index.intersection(benchmark_returns.index)
                                        if common_index.empty:
                                            st.error("No overlapping dates between portfolio returns and benchmark returns after alignment.")
                                            st.stop()
                                        returns = returns.loc[common_index]
                                        cum_returns = cum_returns.loc[common_index]
                                        # Align the data using an inner join
                                        aligned_data = pd.merge(returns.to_frame('Portfolio'), benchmark_returns.to_frame('Benchmark'), left_index=True, right_index=True, how='inner')

                                        # Check that we have data after merging
                                        if aligned_data.empty:
                                            st.error("No overlapping dates between portfolio returns and benchmark returns after alignment.")
                                            st.stop()

                                        # Assign aligned returns
                                        returns = aligned_data['Portfolio'].rename("Portfolio_Returns")
                                        benchmark_returns = aligned_data['Benchmark']
                                        benchmark_cum_returns = (1 + benchmark_returns).cumprod()
                                        benchmark_metrics = calculate_simple_metrics(
                                            benchmark_returns, benchmark_cum_returns
                                        )
                                        extended_benchmark_metrics = calculate_portfolio_metrics(
                                            benchmark_returns, benchmark_cum_returns, portfolio['rf_rate'], None
                                        )
                                        benchmark_metrics.update(extended_benchmark_metrics)
                                    else:
                                        benchmark_returns = None
                                        benchmark_cum_returns = pd.Series(dtype=float)
                                        benchmark_metrics = {
                                            'Start Balance': "N/A",
                                            'End Balance': "N/A",
                                            'Annualized Return (CAGR)': "N/A",
                                            'Best Year': "N/A",
                                            'Worst Year': "N/A",
                                            'Arithmetic Mean (Monthly)': "N/A",
                                            'Arithmetic Mean (Annualized)': "N/A",
                                            'Geometric Mean (Monthly)': "N/A",
                                            'Geometric Mean (Annualized)': "N/A",
                                            'Standard Deviation (Monthly)': "N/A",
                                            'Standard Deviation (Annualized)': "N/A",
                                            'Downside Deviation (Monthly)': "N/A",
                                            'Maximum Drawdown': "N/A",
                                            'Sharpe Ratio': "N/A",
                                            'Sortino Ratio': "N/A",
                                            'Gain/Loss Ratio': "N/A",
                                            'Skewness': "N/A",
                                            'Excess Kurtosis': "N/A",
                                            'Safe Withdrawal Rate': "N/A",
                                            'Perpetual Withdrawal Rate': "N/A",
                                            'Positive Periods': "N/A",
                                            'Benchmark Correlation': "N/A",
                                            'Beta': "N/A",
                                            'Alpha (annualized)': "N/A",
                                            'R2': "N/A",
                                            'Treynor Ratio': "N/A",
                                            'Calmar Ratio': "N/A",
                                            'Modigliani–Modigliani Measure': "N/A",
                                            'Information Ratio': "N/A",
                                            'Tracking Error': "N/A",
                                            'Active Return': "N/A",
                                            'Upside Capture Ratio': "N/A",
                                            'Downside Capture Ratio': "N/A"
                                        }

                        if not cum_returns.empty:
                            portfolio_metrics = calculate_portfolio_metrics(
                                returns, cum_returns, portfolio['rf_rate'], benchmark_returns
                            )

                            # Dashboard Header with Key Stats
                            with st.container():
                                st.markdown("### 🔑 Key Metrics")
                                key_metrics = {
                                    'Annualized Return (CAGR)': portfolio_metrics.get('Annualized Return (CAGR)', "N/A"),
                                    'Sharpe Ratio': portfolio_metrics.get('Sharpe Ratio', "N/A"),
                                    'Maximum Drawdown': portfolio_metrics.get('Maximum Drawdown', "N/A")
                                }
                                cols = st.columns(len(key_metrics))
                                for col, (metric, value) in zip(cols, key_metrics.items()):
                                    with col:
                                        st.metric(label=metric, value=value)

                            # Create Tabs for Organized Sections
                            tabs = st.tabs(["Overview", "Performance Statistics", "Advanced Metrics", "Drawdowns", "Visualizations"])

                            with tabs[0]:
                                st.header("📈 Portfolio Performance Overview")
                                st.write(f"**Portfolio Name:** {portfolio['name']}")
                                st.write(f"**Start Date:** {portfolio['start_date'].strftime('%Y-%m-%d')}")
                                st.write(f"**End Date:** {portfolio['end_date'].strftime('%Y-%m-%d')}")
                                st.write(f"**Benchmark:** {portfolio['benchmark_symbol']}")

                            with tabs[1]:
                                st.header("📊 Performance Statistics")
                                
                                with st.container():
                                    st.subheader("🛠️ Basic Metrics")
                                    performance_data_stats = {
                                        'Metric': [
                                            'Start Balance',
                                            'End Balance',
                                            'Annualized Return (CAGR)',
                                            'Standard Deviation (Annualized)',
                                            'Best Year',
                                            'Worst Year',
                                            'Maximum Drawdown',
                                            'Sharpe Ratio',
                                            'Sortino Ratio',
                                            'Benchmark Correlation'
                                        ],
                                        'Portfolio': [
                                            portfolio_metrics['Start Balance'],
                                            portfolio_metrics['End Balance'],
                                            portfolio_metrics['Annualized Return (CAGR)'],
                                            portfolio_metrics['Standard Deviation (Annualized)'],
                                            portfolio_metrics['Best Year'],
                                            portfolio_metrics['Worst Year'],
                                            portfolio_metrics['Maximum Drawdown'],
                                            portfolio_metrics['Sharpe Ratio'],
                                            portfolio_metrics['Sortino Ratio'],
                                            portfolio_metrics['Benchmark Correlation']
                                        ],
                                        'Benchmark': [
                                            benchmark_metrics['Start Balance'],
                                            benchmark_metrics['End Balance'],
                                            benchmark_metrics['Annualized Return (CAGR)'],
                                            benchmark_metrics['Standard Deviation (Annualized)'],
                                            benchmark_metrics['Best Year'],
                                            benchmark_metrics['Worst Year'],
                                            benchmark_metrics['Maximum Drawdown'],
                                            benchmark_metrics['Sharpe Ratio'],
                                            benchmark_metrics['Sortino Ratio'],
                                            benchmark_metrics['Benchmark Correlation']
                                        ]
                                    }
                                    performance_df_stats = pd.DataFrame(performance_data_stats).set_index('Metric')
                                    st.table(performance_df_stats)

                            with tabs[2]:
                                st.header("📋 Detail Comparisons")
                                
                                with st.container():
                                    st.subheader("🧮 Advanced Metrics")
                                    performance_data_advanced = {
                                        'Metric': list(portfolio_metrics.keys()),
                                        'Portfolio': list(portfolio_metrics.values()),
                                        'Benchmark': list(benchmark_metrics.values())
                                    }
                                    performance_df_advanced = pd.DataFrame(performance_data_advanced).set_index('Metric')

                                    # Replace NaN with empty strings for better visualization
                                    performance_df_advanced = performance_df_advanced.replace(np.nan, "")
                                    # Drop rows where both Portfolio and Benchmark values are NaN
                                    performance_df_advanced_clean = performance_df_advanced.dropna(how='all')

                                    st.table(performance_df_advanced_clean)
                                    st.markdown("---")
                                    st.write("**Note:** Some metrics in the benchmark column are empty because they are portfolio-specific measurements that can't be calculated for the benchmark alone. These include:\n\n"
         "1. Alpha (annualized)\n"
         "2. Information Ratio\n"
         "3. Tracking Error\n"
         "4. Active Return\n"
         "5. Upside/Downside Capture Ratios\n\n"
         "These metrics specifically measure how a portfolio performs relative to its benchmark, so they only make sense when calculated for the portfolio itself. For example, tracking error measures how closely a portfolio follows its benchmark, which isn't applicable to the benchmark itself. Similarly, alpha measures excess return relative to the benchmark, which wouldn't be meaningful to calculate for the benchmark itself.")
                                st.subheader("🔍 Risk Factor Attribution Analysis")
                                
                                # Ensure 'returns' is defined and is a pd.Series
                                if 'returns' not in locals():
                                    st.error("Portfolio returns data ('returns') is not defined.")
                                else:
                                    # Define date range based on portfolio returns index
                                    start_date = returns.index.min().strftime('%Y-%m-%d')
                                    end_date = returns.index.max().strftime('%Y-%m-%d')
                                    # Fetch actual factor data
                                    factors = fetch_fama_french_factors(start_date, end_date)
                                    # Ensure factors are fetched
                                    if factors.empty:
                                        st.write("Failed to retrieve factor data.")       
                                    factors = factors.asfreq(returns.index.freq, method='ffill')
                                    # Calculate Attribution with aligned dates
                                    attribution, error = calculate_risk_factor_attribution(returns, factors)
                                    
                                    if error:
                                        st.error(error)
                                        st.write("Risk factor attribution analysis is unavailable.")
                                    elif attribution.empty:
                                        st.write("Risk factor attribution analysis is unavailable.")
                                    else:
                                        st.write("Risk Factor Attribution:")
                                        st.table(attribution)
                                        # Optional: Visualize Attribution
                                        fig_attribution = px.bar(
                                            attribution,
                                            x='Factor',
                                            y='Contribution (%)',
                                            title='Risk Factor Attribution',
                                            labels={'Contribution (%)': 'Contribution (%)'},
                                            template='plotly_dark'
                                        )
                                        st.plotly_chart(fig_attribution, use_container_width=True)

                            with tabs[3]:
                                st.header("📉 Detailed Drawdowns")
                                st.markdown("### 📈 Drawdowns for Portfolio")
                                portfolio_drawdowns = get_drawdown_details(cum_returns)
                                if portfolio_drawdowns:
                                    portfolio_drawdowns_df = pd.DataFrame(portfolio_drawdowns)
                                    st.table(portfolio_drawdowns_df)
                                else:
                                    st.write("No drawdowns detected for the portfolio.")

                                if benchmark_returns is not None and not benchmark_returns.empty:
                                    st.markdown("### 📈 Drawdowns for Benchmark")
                                    benchmark_drawdowns = get_drawdown_details(benchmark_cum_returns)
                                    if benchmark_drawdowns:
                                        benchmark_drawdowns_df = pd.DataFrame(benchmark_drawdowns)
                                        st.table(benchmark_drawdowns_df)
                                    else:
                                        st.write("No drawdowns detected for the benchmark.")
                                else:
                                    st.write("Benchmark data not available for drawdown analysis.")

                            with tabs[4]:
                                st.header("📊 Visualizations")
                                if not cum_returns.empty and not benchmark_cum_returns.empty:
                                    plot_growth_comparison(cum_returns, benchmark_cum_returns)
                                else:
                                    st.warning("Insufficient data to display Growth Comparison. Ensure both portfolio and benchmark have data.")
                                
                                portfolio_drawdown_series = (cum_returns / cum_returns.expanding().max() - 1) * 100
                                benchmark_drawdown_series = (benchmark_cum_returns / benchmark_cum_returns.expanding().max() - 1) * 100
                                plot_drawdown_comparison(portfolio_drawdown_series, benchmark_drawdown_series)

                                st.subheader("📈 Compound Annual Growth Rate (CAGR) Over Time")
                                if st.session_state.selected_time_frames:
                                    plot_cagr_over_time(cum_returns, time_frames=st.session_state.selected_time_frames)
                                else:
                                    st.warning("Please select at least one time frame for CAGR.")

                                # Box Plot for Returns Distribution
                                st.subheader("📦 Returns Distribution Box Plot")
                                if isinstance(returns, pd.DataFrame):
                                    melted_returns = returns.reset_index().melt(id_vars='Date', var_name='Asset', value_name='Return')
                                    fig_box_plot = px.box(
                                        melted_returns,
                                        x='Asset',
                                        y='Return',
                                        title='Returns Distribution Box Plot',
                                        labels={'Return': 'Returns', 'Asset': 'Asset'},
                                        points='all',
                                        hover_data=['Return'],
                                        template='plotly_dark',
                                        color='Asset',
                                        color_discrete_sequence=px.colors.qualitative.Pastel
                                    )
                                    st.plotly_chart(fig_box_plot, use_container_width=True)
                                    st.markdown("**Interpretation:** The box plot visualizes the distribution of returns for each asset in the portfolio. The boxes represent the interquartile range (IQR), the line inside the box indicates the median, and the whiskers show the range of the data. Outliers are displayed as individual points.")
                                elif isinstance(returns, pd.Series):
                                    fig_box_plot = px.box(
                                        returns.to_frame(name='Return'),
                                        y='Return',
                                        title='Returns Distribution Box Plot',
                                        labels={'Return': 'Returns'},
                                        points='all',
                                        hover_data=['Return'],
                                        template='plotly_dark',
                                        color_discrete_sequence=px.colors.qualitative.Pastel
                                    )
                                    st.plotly_chart(fig_box_plot, use_container_width=True)
                                    st.markdown("**Interpretation:** The box plot visualizes the distribution of returns for each asset in the portfolio. The boxes represent the interquartile range (IQR), the line inside the box indicates the median, and the whiskers show the range of the data. Outliers are displayed as individual points.")
                                else:
                                    st.warning("Returns data is neither a DataFrame nor a Series.")

                                # Heatmap of Correlations Between Assets
                                st.subheader("🔥 Correlation Heatmap of Assets")
                                plot_correlation_heatmap(price_data[available_selected].pct_change().dropna())
                                
                                # Add Risk-Return Attribution Analysis
                                st.subheader("🔍 Risk-Return Attribution Analysis")
                                plot_risk_return_attribution(returns, weights)
                                
                                # Cumulative Returns Heatmap
                                st.subheader("🔥 Cumulative Returns Heatmap")
                                try:
                                    cum_returns_normalized = cum_returns / cum_returns.max()
                                    fig_cum_heatmap = px.imshow(
                                        cum_returns_normalized.to_frame().T,
                                        labels=dict(x="Date", y="Portfolio", color="Normalized Cumulative Return"),
                                        title="Cumulative Returns Heatmap",
                                        aspect="auto",
                                        color_continuous_scale='Viridis',
                                        template='plotly_dark'
                                    )
                                    st.plotly_chart(fig_cum_heatmap, use_container_width=True)
                                    st.markdown("**Interpretation:** The heatmap visualizes the normalized cumulative returns over time, allowing for an intuitive comparison of portfolio performance across different periods.")
                                except Exception as e:
                                    st.error(f"Error plotting cumulative returns heatmap: {e}")

                                # Portfolio Allocation Pie Chart
                                st.subheader("🥧 Portfolio Allocation Pie Chart")
                                plot_allocation_pie(weights, available_selected, title="Portfolio Allocation", hover_info="percent+name")
                                
                                # Rolling Metrics Chart with User-Defined Periods
                                st.subheader("📉 Rolling Metrics")
                                if st.session_state.selected_rolling_periods:
                                    plot_rolling_metrics(returns, windows=st.session_state.selected_rolling_periods)
                                else:
                                    st.warning("Please select at least one rolling period to display metrics.")

                                # Recommendations
                                st.markdown("### 💡 Recommendations")
                                recommendations = generate_recommendations(
                                    allocations=[alloc for alloc in portfolio['allocations'] if alloc > 0],
                                    available_selected=[ticker for ticker in available_selected if portfolio['allocations'][available_selected.index(ticker)] > 0]
                                )
                                
                                if recommendations:
                                    for rec in recommendations:
                                        st.write(rec)
                                else:
                                    st.success("Your portfolio allocations are well-balanced!")
                                    
                                # Final Score
                                st.markdown("### 📝 Final Score")
                                portfolio_score = calculate_final_score(portfolio_metrics, benchmark_metrics)
                                benchmark_score = calculate_final_score(benchmark_metrics, portfolio_metrics)
                                st.write(f"**Portfolio:** {portfolio_score:.2f} / 100")
                                st.write(f"**Benchmark:** {benchmark_score:.2f} / 100")

                                # Store results
                                st.session_state.backtest_results = {
                                    'returns': returns,
                                    'cum_returns': cum_returns,
                                    'weights_a': weights,
                                    'price_data': price_data[available_selected],
                                    'metrics': portfolio_metrics,
                                    'benchmark_metrics': benchmark_metrics,
                                    'performance_df_stats': performance_df_stats,
                                    'performance_df_advanced': performance_df_advanced,
                                    'benchmark_cum_returns': benchmark_cum_returns
                                }
                                st.success("Backtest completed!")

                elif comparison_type == "Portfolio vs Portfolio":
                    if portfolio1 == portfolio2:
                        st.error("Please select two different portfolios for comparison.")
                    else:
                        portfolio_a = next(p for p in st.session_state.portfolios if p['name'] == portfolio1)
                        portfolio_b = next(p for p in st.session_state.portfolios if p['name'] == portfolio2)

                        overlap_start = max(portfolio_a['start_date'], portfolio_b['start_date'])
                        overlap_end = min(portfolio_a['end_date'], portfolio_b['end_date'])

                        if overlap_start >= overlap_end:
                            st.error("The selected portfolios do not have overlapping date ranges.")
                            st.stop()

                        price_data_a = download_data(
                            portfolio_a['selected'],
                            overlap_start,
                            overlap_end
                        )
                        price_data_b = download_data(
                            portfolio_b['selected'],
                            overlap_start,
                            overlap_end
                        )

                        if price_data_a.empty:
                            st.error(f"No price data available for portfolio '{portfolio_a['name']}' in the overlapping period.")
                            st.stop()
                        if price_data_b.empty:
                            st.error(f"No price data available for portfolio '{portfolio_b['name']}' in the overlapping period.")
                            st.stop()

                        # Backtest Portfolio A
                        available_a = [ticker for ticker in portfolio_a['selected'] if ticker in price_data_a.columns]
                        missing_a = list(set(portfolio_a['selected']) - set(available_a))
                        if missing_a:
                            st.warning(f"Excluded tickers from '{portfolio_a['name']}': {', '.join(missing_a)}")
                        if not available_a:
                            st.error(f"No valid tickers for portfolio '{portfolio_a['name']}' in the overlapping period.")
                            st.stop()
                        allocs_a = [portfolio_a['allocations'][i] for i, ticker in enumerate(portfolio_a['selected']) if ticker in available_a]
                        if not np.isclose(sum(allocs_a), 100.0, atol=1e-4):
                            st.warning(f"Allocations for '{portfolio_a['name']}' do not sum to 100%. Adjusting allocations proportionally.")
                            allocs_a = [a / sum(allocs_a) * 100 for a in allocs_a]
                        weights_a = np.array(allocs_a) / 100
                        returns_a, cum_returns_a = backtest(weights_a, price_data_a[available_a],
                                                           portfolio_a['rebalance_freq'], portfolio_a['broker_fee'], debug=False)

                        # Backtest Portfolio B
                        available_b = [ticker for ticker in portfolio_b['selected'] if ticker in price_data_b.columns]
                        missing_b = list(set(portfolio_b['selected']) - set(available_b))
                        if missing_b:
                            st.warning(f"Excluded tickers from '{portfolio_b['name']}': {', '.join(missing_b)}")
                        if not available_b:
                            st.error(f"No valid tickers for portfolio '{portfolio_b['name']}' in the overlapping period.")
                            st.stop()
                        allocs_b = [portfolio_b['allocations'][i] for i, ticker in enumerate(portfolio_b['selected']) if ticker in available_b]
                        if not np.isclose(sum(allocs_b), 100.0, atol=1e-4):
                            st.warning(f"Allocations for '{portfolio_b['name']}' do not sum to 100%. Adjusting allocations proportionally.")
                            allocs_b = [a / sum(allocs_b) * 100 for a in allocs_b]
                        weights_b = np.array(allocs_b) / 100
                        returns_b, cum_returns_b = backtest(weights_b, price_data_b[available_b],
                                                           portfolio_b['rebalance_freq'], portfolio_b['broker_fee'], debug=False)

                        if cum_returns_a.empty or cum_returns_b.empty:
                            st.error("One of the portfolios has empty cumulative returns. Check the data and allocations.")
                            st.stop()

                        # Align date ranges between the two portfolios
                        common_index = cum_returns_a.index.intersection(cum_returns_b.index)
                        if common_index.empty:
                            st.error("No overlapping dates between the two portfolios after backtesting.")
                            st.stop()
                        returns_a = returns_a.loc[common_index]
                        cum_returns_a = cum_returns_a.loc[common_index]
                        returns_b = returns_b.loc[common_index]
                        cum_returns_b = cum_returns_b.loc[common_index]

                        # Calculate metrics without using one portfolio as the benchmark for the other
                        metrics_a = calculate_portfolio_metrics(
                            returns_a, cum_returns_a, portfolio_a['rf_rate'], benchmark_returns=None
                        )
                        metrics_b = calculate_portfolio_metrics(
                            returns_b, cum_returns_b, portfolio_b['rf_rate'], benchmark_returns=None
                        )

                        # Dashboard Header with Key Stats
                        with st.container():
                            st.markdown("### 🔑 Key Metrics")
                            key_metrics_a = {
                                'Annualized Return (CAGR)': metrics_a.get('Annualized Return (CAGR)', "N/A"),
                                'Sharpe Ratio': metrics_a.get('Sharpe Ratio', "N/A"),
                                'Maximum Drawdown': metrics_a.get('Maximum Drawdown', "N/A")
                            }
                            key_metrics_b = {
                                'Annualized Return (CAGR)': metrics_b.get('Annualized Return (CAGR)', "N/A"),
                                'Sharpe Ratio': metrics_b.get('Sharpe Ratio', "N/A"),
                                'Maximum Drawdown': metrics_b.get('Maximum Drawdown', "N/A")
                            }
                            cols = st.columns(len(key_metrics_a))
                            for col, (metric, value) in zip(cols, key_metrics_a.items()):
                                with col:
                                    st.metric(label=f"{portfolio_a['name']} {metric}", value=value)
                            for col, (metric, value) in zip(cols, key_metrics_b.items()):
                                with col:
                                    st.metric(label=f"{portfolio_b['name']} {metric}", value=value)

                        # Create Tabs for Organized Sections
                        tabs = st.tabs(["Overview", "Performance Statistics", "Advanced Metrics", "Drawdowns", "Visualizations"])

                        with tabs[0]:
                            st.header("📈 Portfolio Performance Overview")
                            st.write(f"**Portfolio A Name:** {portfolio_a['name']}")
                            st.write(f"**Portfolio B Name:** {portfolio_b['name']}")
                            st.write(f"**Start Date:** {portfolio_a['start_date']}")
                            st.write(f"**End Date:** {portfolio_a['end_date']}")

                        with tabs[1]:
                            st.header("📊 Performance Statistics")
                            with st.expander("🛠️ Basic Metrics", expanded=True):
                                performance_data_stats = {
                                    'Metric': [
                                        'Start Balance',
                                        'End Balance',
                                        'Annualized Return (CAGR)',
                                        'Standard Deviation (Annualized)',
                                        'Best Year',
                                        'Worst Year',
                                        'Maximum Drawdown',
                                        'Sharpe Ratio',
                                        'Sortino Ratio',
                                        'Benchmark Correlation'
                                    ],
                                    'Portfolio': [
                                        portfolio_metrics['Start Balance'],
                                        portfolio_metrics['End Balance'],
                                        portfolio_metrics['Annualized Return (CAGR)'],
                                        portfolio_metrics['Standard Deviation (Annualized)'],
                                        portfolio_metrics['Best Year'],
                                        portfolio_metrics['Worst Year'],
                                        portfolio_metrics['Maximum Drawdown'],
                                        portfolio_metrics['Sharpe Ratio'],
                                        portfolio_metrics['Sortino Ratio'],
                                        portfolio_metrics['Benchmark Correlation']
                                    ],
                                    'Benchmark': [
                                        benchmark_metrics['Start Balance'],
                                        benchmark_metrics['End Balance'],
                                        benchmark_metrics['Annualized Return (CAGR)'],
                                        benchmark_metrics['Standard Deviation (Annualized)'],
                                        benchmark_metrics['Best Year'],
                                        benchmark_metrics['Worst Year'],
                                        benchmark_metrics['Maximum Drawdown'],
                                        benchmark_metrics['Sharpe Ratio'],
                                        benchmark_metrics['Sortino Ratio'],
                                        benchmark_metrics['Benchmark Correlation']
                                    ]
                                }

                                performance_df_stats = pd.DataFrame(performance_data_stats).set_index('Metric')

                                # Replace NaN with empty strings for better visualization
                                performance_df_stats = performance_df_stats.replace(np.nan, "")

                                st.table(performance_df_stats)

                        with tabs[2]:
                            st.header("📉 Advanced Metrics")
                            with st.expander("📈 Advanced Metrics", expanded=False):
                                performance_data_advanced = {
                                    'Metric': list(metrics_a.keys()),
                                    portfolio_a['name']: list(metrics_a.values()),
                                    portfolio_b['name']: list(metrics_b.values())
                                }
                                performance_df_advanced = pd.DataFrame(performance_data_advanced).set_index('Metric')
                                st.table(performance_df_advanced)

                        with tabs[3]:
                            st.header("📉 Drawdowns")
                            st.markdown("### 📉 Detailed Drawdowns")
                            st.markdown(f"#### 📉 Drawdowns for {portfolio_a['name']}")
                            drawdowns_a = get_drawdown_details(cum_returns_a)
                            if drawdowns_a:
                                drawdowns_a_df = pd.DataFrame(drawdowns_a)
                                st.table(drawdowns_a_df)
                            else:
                                st.write("No drawdowns detected.")

                            st.markdown(f"#### 📉 Drawdowns for {portfolio_b['name']}")
                            drawdowns_b = get_drawdown_details(cum_returns_b)
                            if drawdowns_b:
                                drawdowns_b_df = pd.DataFrame(drawdowns_b)
                                st.table(drawdowns_b_df)
                            else:
                                st.write("No drawdowns detected.")

                        with tabs[4]:
                            st.header("📊 Visualizations")
                            # Growth Comparison
                            fig_growth = px.line(title='Growth Comparison')
                            fig_growth.add_scatter(x=cum_returns_a.index, y=cum_returns_a, mode='lines', name=portfolio_a['name'])
                            fig_growth.add_scatter(x=cum_returns_b.index, y=cum_returns_b, mode='lines', name=portfolio_b['name'])
                            st.plotly_chart(fig_growth, use_container_width=True)

                            # Drawdown Comparison
                            portfolio_drawdown_series_a = (cum_returns_a / cum_returns_a.expanding().max() - 1) * 100
                            portfolio_drawdown_series_b = (cum_returns_b / cum_returns_b.expanding().max() - 1) * 100
                            fig_drawdown = px.line(title='Drawdown Comparison')
                            fig_drawdown.add_scatter(x=portfolio_drawdown_series_a.index, y=portfolio_drawdown_series_a, mode='lines', name=portfolio_a['name'])
                            fig_drawdown.add_scatter(x=portfolio_drawdown_series_b.index, y=portfolio_drawdown_series_b, mode='lines', name=portfolio_b['name'])
                            st.plotly_chart(fig_drawdown, use_container_width=True)

                            # Box Plot for Returns Distribution
                            st.subheader("📦 Returns Distribution Box Plot")
                            fig_box_plot_a = px.box(returns_a, title=f'Returns Distribution Box Plot for {portfolio_a["name"]}', labels={'value': 'Returns', 'variable': 'Asset'})
                            fig_box_plot_b = px.box(returns_b, title=f'Returns Distribution Box Plot for {portfolio_b["name"]}', labels={'value': 'Returns', 'variable': 'Asset'})
                            st.plotly_chart(fig_box_plot_a, use_container_width=True)
                            st.plotly_chart(fig_box_plot_b, use_container_width=True)

                            # Heatmap of Correlations Between Assets
                            st.subheader("🔥 Correlation Heatmap of Assets")
                            plot_correlation_heatmap(price_data_a[available_a].pct_change().dropna())
                            plot_correlation_heatmap(price_data_b[available_b].pct_change().dropna())

                            # Portfolio Allocation Pie Chart
                            st.subheader("🥧 Portfolio Allocation Pie Chart")
                            plot_allocation_pie(weights_a, available_a, title=f"{portfolio_a['name']} Allocation", hover_info="percent+name")
                            plot_allocation_pie(weights_b, available_b, title=f"{portfolio_b['name']} Allocation", hover_info="percent+name")

                            # Final Score
                            st.markdown("### 📝 Final Score")
                            score_a = calculate_final_score(metrics_a, metrics_b)
                            score_b = calculate_final_score(metrics_b, metrics_a)
                            st.write(f"**{portfolio_a['name']}:** {score_a:.2f} / 100")
                            st.write(f"**{portfolio_b['name']}:** {score_b:.2f} / 100")

                            # Store results
                            st.session_state.backtest_results = {
                                'returns_a': returns_a,
                                'cum_returns_a': cum_returns_a,
                                'weights_a': weights_a,
                                'price_data_a': price_data_a[available_a],
                                'metrics_a': metrics_a,
                                'returns_b': returns_b,
                                'cum_returns_b': cum_returns_b,
                                'weights_b': weights_b,
                                'price_data_b': price_data_b[available_b],
                                'metrics_b': metrics_b,
                                'performance_df_stats': performance_df_stats,
                                'performance_df_advanced': performance_df_advanced,
                                'benchmark_cum_returns': pd.Series(dtype=float)
                            }
                            st.success("Backtest completed!")
    else:
        st.warning("Please add at least one portfolio to run backtest.")

    # ----------------------------
    # Onboarding Process
    # ----------------------------
    if 'first_time_user' not in st.session_state:
        st.session_state.first_time_user = True

    if st.session_state.first_time_user:
        st.info("Welcome! Let's get you started with your first portfolio.")
        st.write("1. Configure your portfolio by entering a name, start date, end date, and other details.")
        st.write("2. Select the assets you want to include and set their allocations.")
        st.write("3. Run a backtest to see how your portfolio performs.")
        st.write("4. Compare your portfolio with a benchmark or another portfolio.")
        st.write("5. Explore the performance metrics and visualizations.")
        st.session_state.first_time_user = False

    # ----------------------------
    # Documentation
    # ----------------------------
    with st.expander("📖 Documentation"):
        st.markdown("""
        ### Documentation

        **Portfolio Configuration:**
        - **Portfolio Name:** Enter a unique name for your portfolio.
        - **Risk-Free Rate:** The theoretical return of an investment with zero risk, often based on government bonds.
        - **Sharpe Ratio:** Measures the performance of an investment compared to a risk-free asset, after adjusting for its risk. 
            *Formula:* $$(R_p - R_f) / \sigma_p$$
        - **VaR (Value at Risk):** Estimates the maximum potential loss over a specific time frame at a given confidence level.
        - **CVaR (Conditional Value at Risk):** The expected loss exceeding the VaR, providing insight into tail risk.
        - **Sortino Ratio:** Similar to the Sharpe Ratio but only penalizes downside volatility.
            *Formula:* $$(R_p - R_f) / \sigma_d$$
        - **Maximum Drawdown:** The largest peak-to-trough decline in the portfolio's value over a specific period.
        - **Alpha:** Measures the active return on an investment compared to a market index.
            *Formula:* $$R_p - [R_f + \beta (R_m - R_f)]$$
        - **Beta:** Indicates the volatility of an investment relative to the market.
            *Formula:* $$\beta = \frac{Cov(R_p, R_m)}{Var(R_m)}$$
        - **Information Ratio:** Measures portfolio returns beyond the returns of a benchmark, adjusted for the volatility of those returns.
            *Formula:* $$(R_p - R_b) / TE$$
        - **Gain/Loss Ratio:** The ratio of total gains to total losses in the portfolio.
        - **Modigliani–Modigliani Measure (M²):** Adjusts the portfolio return to the risk of a benchmark, allowing for comparison.
            *Formula:* $$\alpha / \text{Sharpe Ratio}$$
        - **Tracking Error:** Measures the standard deviation of the difference between portfolio returns and benchmark returns.
        - **Upside/Downside Capture Ratio:** Measures how well the portfolio captures the benchmark's positive and negative movements respectively.
        - **Risk Factor Attribution:** Decomposes portfolio returns based on exposure to different risk factors like Momentum, Value, and Size.
        - **Performance Attribution:** Breaks down portfolio performance by individual assets, showing each asset's contribution to overall returns.
        - **Recommendation Engine:** Provides suggestions for portfolio adjustments to optimize performance based on current allocations and objectives.
        """)

elif step == "Optimize Portfolio":
    st.title("🔧 Optimize Portfolio")
    if st.session_state.portfolios:
        portfolio_names = [p['name'] for p in st.session_state.portfolios]
        selected_portfolio = st.selectbox("Select Portfolio to Optimize", portfolio_names)

        portfolio = next(p for p in st.session_state.portfolios if p['name'] == selected_portfolio)
        price_data = download_data(
            portfolio['selected'],
            portfolio['start_date'],
            portfolio['end_date']
        )
        if price_data.empty:
            st.error("No price data available for the selected portfolio.")
        else:
            available_selected = [ticker for ticker in portfolio['selected'] if ticker in price_data.columns]
            missing_selected = list(set(portfolio['selected']) - set(available_selected))
            if missing_selected:
                st.warning(f"Excluded tickers with no data: {', '.join(missing_selected)}")
            if not available_selected:
                st.error(
                    "No selected tickers have available data for the chosen date range. "
                    "Please ensure that the ticker symbols are correct and data is available for the specified period."
                )
            else:
                returns = price_data[available_selected].pct_change().dropna()
                with st.form("optimization_form"):
                    st.markdown("### 🛠️ Set Optimization Constraints")
                    # Interactive Sliders
                    max_weight = st.slider(
                        "Maximum Weight per Asset (%)",
                        min_value=0.0,
                        max_value=100.0,
                        value=100.0,
                        step=0.1,
                        help="Set the maximum allocation percentage for any single asset."
                    ) / 100  # Convert to decimal

                    min_weight = st.slider(
                        "Minimum Weight per Asset (%)",
                        min_value=0.0,
                        max_value=100.0,
                        value=0.0,
                        step=0.1,
                        help="Set the minimum allocation percentage for any single asset."
                    ) / 100  # Convert to decimal

                    # **Added Validation Check**
                    if min_weight > max_weight:
                        st.error("Minimum weight cannot exceed maximum weight.")
                        st.stop()  # Prevent further execution

                    # **Added: Multi-Objective Selection**
                    objective_options = [
                        "Sharpe Ratio", 
                        "Minimum Variance", 
                        "Maximum Return", 
                        "Minimum Drawdown",
                        "Maximize Alpha",
                        "Minimize Beta"
                    ]
                    selected_objectives = st.multiselect(
                        "Select Optimization Objectives",
                        options=objective_options,
                        default=["Sharpe Ratio"],
                        help="Choose one or more objectives to optimize your portfolio."
                    )

                    target_return = st.number_input(
                        "Target Annual Return (%)",
                        min_value=0.0,
                        max_value=100.0,
                        value=10.0,
                        step=0.1,
                        help="Set the target annual return for portfolio optimization."
                    ) / 100  # Convert to decimal

                    submitted_optimize = st.form_submit_button("Run Optimization")

                    if submitted_optimize:
                        if not selected_objectives:
                            st.error("Please select at least one optimization objective.")
                        objective_map = {
                            'Sharpe Ratio': 'sharpe',
                            'Minimum Variance': 'min_variance',
                            'Maximum Return': 'max_return',
                            'Minimum Drawdown': 'min_drawdown',
                            'Maximize Alpha': 'maximize_alpha',       # Added entry
                            'Minimize Beta': 'minimize_beta'          # Added entry
                        }
                        objectives_selected = [objective_map[obj] for obj in selected_objectives]
                        with st.spinner("Optimizing portfolio..."):
                            # Download benchmark data
                            benchmark_symbol = portfolio['benchmark_symbol']
                            benchmark_data = download_data([benchmark_symbol], portfolio['start_date'], portfolio['end_date'])
                            if benchmark_data.empty:
                                st.error(f"Benchmark symbol '{benchmark_symbol}' does not have available data for the chosen period.")
                                st.stop()
                            else:
                                benchmark_returns = benchmark_data[benchmark_symbol].pct_change().dropna()
                                benchmark_returns = benchmark_returns.reindex(returns.index, method='ffill').dropna()

                            # Call optimize_portfolio with benchmark_returns
                            optimized_weights = optimize_portfolio(
                                returns,
                                benchmark_returns=benchmark_returns,  # Passed as a new argument
                                objectives=objectives_selected,
                                rf=portfolio['rf_rate'],
                                max_weight=max_weight,
                                min_weight=min_weight,
                                target_return=target_return
                            )
                            if optimized_weights is not None:
                                # **Added Allocation Sum Check**
                                total_allocation = np.sum(optimized_weights)
                                if not np.isclose(total_allocation, 1.0, atol=1e-4):
                                    st.warning(f"Optimized allocations sum to {total_allocation*100:.2f}%. Adjusting proportionally.")
                                    optimized_weights = optimized_weights / total_allocation

                                st.success("Optimization completed successfully!")

                                # Display Optimized Weights
                                st.subheader("📈 Optimized Weights")
                                weights_df = pd.DataFrame(optimized_weights, index=available_selected, columns=["Weight"])
                                weights_df['Weight'] = weights_df['Weight'].apply(lambda x: f"{x:.2%}")
                                st.table(weights_df)

                                # Plot Allocation Pie Chart
                                plot_allocation_pie(optimized_weights, available_selected, title="Optimized Portfolio Allocation")

                                # Display Performance Metrics
                                st.subheader("📈 Optimized Portfolio Performance Metrics")
                                optimized_return = np.dot(optimized_weights, returns.mean()) * 252
                                optimized_volatility = np.sqrt(np.dot(optimized_weights.T, np.dot(returns.cov() * 252, optimized_weights)))
                                optimized_sharpe = (optimized_return - portfolio['rf_rate']) / optimized_volatility if optimized_volatility > 1e-6 else np.nan

                                optimized_metrics = {
                                    'Expected Annual Return (%)': f"{optimized_return * 100:.2f}%",
                                    'Annualized Volatility (%)': f"{optimized_volatility * 100:.2f}%",
                                    'Sharpe Ratio': f"{optimized_sharpe:.2f}"
                                }

                                optimized_metrics_df = pd.DataFrame(list(optimized_metrics.items()), columns=['Metric', 'Value'])
                                st.table(optimized_metrics_df)

                                # Store optimized weights and metrics in session state
                                st.session_state.backtest_results.update({
                                    'optimized_weights': optimized_weights,
                                    'optimized_return': optimized_return,
                                    'optimized_volatility': optimized_volatility,
                                    'optimized_sharpe': optimized_sharpe
                                })
                            else:
                                st.error("Optimization did not return any weights.")
    else:
        st.warning("Please add at least one portfolio to optimize.")

elif step == "Monte Carlo Simulations":
    st.title("📈 Monte Carlo Simulations")
    if st.session_state.portfolios:
        portfolio_names = [p['name'] for p in st.session_state.portfolios]
        selected_portfolio = st.selectbox("Select Portfolio for Monte Carlo Simulations", portfolio_names)
        if selected_portfolio:
            portfolio = next(p for p in st.session_state.portfolios if p['name'] == selected_portfolio)
            price_data = download_data(
                portfolio['selected'],
                portfolio['start_date'],
                portfolio['end_date']
            )
            if price_data.empty:
                st.error("No price data available for the selected portfolio.")
            else:
                available_selected = [ticker for ticker in portfolio['selected'] if ticker in price_data.columns]
                missing_selected = list(set(portfolio['selected']) - set(available_selected))
                if missing_selected:
                    st.warning(f"Excluded tickers with no data: {', '.join(missing_selected)}")
                
                if not available_selected:
                    st.error("No selected tickers have available data for the chosen date range.")
                else:
                    returns = price_data[available_selected].pct_change().dropna()
                    
                    # Organize simulation settings and results into tabs
                    simulation_tabs = st.tabs(["🔧 Settings", "📊 Results"])
                    
                    with simulation_tabs[0]:
                        st.subheader("🔍 Select Simulation Parameters")
                        with st.form("mc_simulation_form"):
                            # Simulation Settings Organized in Columns
                            sim_col1, sim_col2 = st.columns(2)
                            
                            with sim_col1:
                                num_simulations = st.number_input("Number of Simulations", min_value=1, max_value=10000, value=1000)
                                periods = st.number_input("Number of Periods", min_value=1, max_value=252, value=252)
                            
                            with sim_col2:
                                return_distribution = st.selectbox(
                                    "Return Distribution",
                                    options=["normal", "log-normal"],
                                    index=0,
                                    help="Select the return distribution to use in simulations."
                                )
                                enable_mean_reversion = st.checkbox("Enable Mean Reversion")
                                if enable_mean_reversion:
                                    mean_reversion_speed = st.slider("Mean Reversion Speed", min_value=0.0, max_value=1.0, value=0.1)
                                    long_term_mean_input = st.number_input("Long-term Mean Return (%)", value=0.0) / 100
                                    long_term_mean = np.full(len(available_selected), long_term_mean_input / 252)
                                else:
                                    mean_reversion_speed = 0.0
                                    long_term_mean = None
                            
                            # Additional Simulation Settings
                            st.subheader("⚙️ Additional Settings")
                            time_varying_vol = st.checkbox("Enable Time-Varying Volatility")
                            if time_varying_vol:
                                vol_change_rate = st.number_input("Volatility Change Rate per Period", value=0.0) / np.sqrt(252)
                            else:
                                vol_change_rate = 0.0
                            
                            # Stress Testing Settings
                            st.subheader("⚠️ Stress Testing")
                            enable_stress_testing = st.checkbox("Enable Stress Testing")
                            stress_shocks = {}
                            if enable_stress_testing:
                                selected_stress_assets = st.multiselect(
                                    "Select Assets to Apply Stress Shocks",
                                    options=available_selected,
                                    help="Choose assets to apply stress shocks."
                                )
                                for ticker in selected_stress_assets:
                                    shock = st.number_input(
                                        f"Shock to {ticker} (%)",
                                        min_value=-100.0, max_value=100.0,
                                        value=0.0,
                                        step=0.1,
                                        key=f"shock_{ticker}"
                                    ) / 100  # Convert to decimal
                                    stress_shocks[ticker] = shock
                            
                            submitted_simulation = st.form_submit_button("Run Simulations")
                        
                        if submitted_simulation:
                            with st.spinner("Running Monte Carlo simulations..."):
                                simulated_returns = monte_carlo_simulation(
                                    returns, num_simulations=num_simulations, periods=periods, 
                                    mean_returns=returns.mean(), cov_matrix=returns.cov(),
                                    mean_reversion=enable_mean_reversion, 
                                    mean_reversion_speed=mean_reversion_speed, 
                                    long_term_mean=long_term_mean,
                                    time_varying_vol=time_varying_vol, 
                                    vol_change_rate=vol_change_rate,
                                    stress_shocks=stress_shocks, 
                                    stress_period=1,  # Example value
                                    return_distribution=return_distribution
                                )
                                st.session_state.simulated_returns = simulated_returns
                                st.success("Simulations completed successfully!")
                    
                    with simulation_tabs[1]:
                        if 'simulated_returns' in st.session_state:
                            simulated_returns = st.session_state.simulated_returns
                            st.subheader("📊 Simulation Results")
                            
                            # Display Basic Statistics
                            st.write("### Descriptive Statistics")
                            st.write(pd.Series(simulated_returns).describe())
                            
                            # Histogram of Simulated Returns
                            st.write("### Returns Distribution")
                            fig_hist = px.histogram(
                                simulated_returns, 
                                nbins=50, 
                                title='Simulated Returns Distribution',
                                labels={'value': 'Simulated Returns', 'count': 'Frequency'},
                                template='plotly_dark'
                            )
                            st.plotly_chart(fig_hist, use_container_width=True)
                            
                            # Cumulative Distribution Function (CDF)
                            st.write("### Cumulative Distribution Function (CDF)")
                            fig_cdf = px.ecdf(
                                simulated_returns, 
                                title='Cumulative Distribution of Simulated Returns',
                                labels={'value': 'Simulated Returns', 'cumcount': 'CDF'},
                                template='plotly_dark'
                            )
                            st.plotly_chart(fig_cdf, use_container_width=True)
                            
                            # Box Plot for Simulated Returns
                            st.write("### Box Plot of Simulated Returns")
                            fig_box = px.box(
                                pd.DataFrame(simulated_returns, columns=['Returns']),
                                y='Returns',
                                title='Box Plot of Simulated Returns',
                                template='plotly_dark'
                            )
                            st.plotly_chart(fig_box, use_container_width=True)
                            
                            # Summary Statistics Table
                            st.write("### Summary Statistics")
                            summary_stats = pd.DataFrame({
                                'Statistic': ['Mean', 'Median', 'Standard Deviation', 'Minimum', 'Maximum'],
                                'Value': [
                                    f"{np.mean(simulated_returns):.4f}",
                                    f"{np.median(simulated_returns):.4f}",
                                    f"{np.std(simulated_returns):.4f}",
                                    f"{np.min(simulated_returns):.4f}",
                                    f"{np.max(simulated_returns):.4f}"
                                ]
                            })
                            st.table(summary_stats)
                            st.markdown("""
                            **Interpretation:**
                            - **Descriptive Statistics:** Provides an overview of the distribution of simulated portfolio returns.
                            - **Returns Distribution Histogram:** Visualizes the frequency of different return outcomes.
                            - **Cumulative Distribution Function (CDF):** Shows the probability that a return is less than or equal to a particular value.
                            - **Box Plot:** Highlights the median, quartiles, and potential outliers in the return distribution.
                            - **Summary Statistics Table:** Summarizes key metrics from the simulation runs.
                            """)
                        else:
                            st.info("Run simulations to view results.")
    else:
        st.warning("Please add at least one portfolio to run Monte Carlo simulations.")

# Main Risk Analysis Section
elif step == "Risk Analysis":
    st.title("⚠️ Risk Analysis")
    if st.session_state.portfolios:
        portfolio_names = [p['name'] for p in st.session_state.portfolios]
        selected_portfolio = st.selectbox("Select Portfolio for Risk Analysis", portfolio_names)
        if selected_portfolio:
            portfolio = next(p for p in st.session_state.portfolios if p['name'] == selected_portfolio)
            price_data = download_data(
                portfolio['selected'],
                portfolio['start_date'],
                portfolio['end_date']
            )
            if price_data.empty:
                st.error("No price data available for the selected portfolio.")
            else:
                available_selected = [ticker for ticker in portfolio['selected'] if ticker in price_data.columns]
                missing_selected = list(set(portfolio['selected']).difference(set(available_selected)))
                if missing_selected:
                    st.warning(f"Excluded tickers with no data: {', '.join(missing_selected)}")
                if not available_selected:
                    st.error("No selected tickers have available data for the chosen date range.")
                else:
                    returns = price_data[available_selected].pct_change().dropna()
                    
                    # Organize risk analysis settings and outputs into tabs
                    risk_tabs = st.tabs(["📉 Risk Metrics", "📈 Visualizations", "🛠️ Sensitivity Analysis"])
                    
                    with risk_tabs[0]:
                        st.subheader("🔍 Risk Metrics")
                        # Start of the form
                        with st.form("risk_analysis_form"):
                            st.subheader("Risk Analysis Parameters")
                            confidence_level = st.slider("Confidence Level", min_value=0.01, max_value=0.99, value=0.95)
                            
                            # Updated Stress Testing Inputs with Asset Selection
                            st.subheader("⚠️ Enhanced Stress Testing")
                            enable_stress_testing = st.checkbox("Enable Enhanced Stress Testing")
                            stress_shocks = {}

                            if enable_stress_testing:
                                selected_stress_assets = st.multiselect(
                                    "Select Assets to Stress Test",
                                    options=available_selected,
                                    help="Choose multiple assets to apply simultaneous stress shocks."
                                )
                                for ticker in selected_stress_assets:
                                    shock = st.number_input(
                                        f"Shock to {ticker} (%)",
                                        min_value=-100.0, max_value=100.0,
                                        value=0.0,
                                        step=0.1,
                                        key=f"enhanced_shock_{ticker}"
                                    ) / 100  # Convert to decimal
                                    stress_shocks[ticker] = shock
                                if not selected_stress_assets:
                                    st.warning("No assets selected for stress testing.")
                            
                            # Collect inputs for Sensitivity Analysis
                            st.subheader("🔎 Sensitivity Analysis")
                            enable_sensitivity_analysis = st.checkbox("Enable Sensitivity Analysis")
                            sensitivity_adjustments = {}
                            if enable_sensitivity_analysis:
                                interest_rate_change = st.number_input("Change in Interest Rates (bps)", value=0.0) / 10000
                                inflation_rate_change = st.number_input("Change in Inflation Rates (bps)", value=0.0) / 10000
                                for ticker in available_selected:
                                    interest_sensitivity = st.number_input(f"Interest Rate Sensitivity for {ticker}", value=1.0, key=f"interest_sens_{ticker}")
                                    inflation_sensitivity = st.number_input(f"Inflation Rate Sensitivity for {ticker}", value=1.0, key=f"inflation_sens_{ticker}")
                                    adjustment = (interest_sensitivity * interest_rate_change +
                                                  inflation_sensitivity * inflation_rate_change)
                                    sensitivity_adjustments[ticker] = adjustment
                            
                            # Add Cross-Asset Sensitivity Analysis Inputs
                            st.subheader("🔄 Cross-Asset Sensitivity Analysis")
                            enable_cross_asset_sensitivity_analysis = st.checkbox("Enable Cross-Asset Sensitivity Analysis")
                            cross_asset_sensitivity_adjustments = {}

                            if enable_cross_asset_sensitivity_analysis:
                                st.markdown("### Currency Sensitivity")
                                currency_rate_change = st.number_input(
                                    "Change in Currency Exchange Rate (%)",
                                    min_value=-50.0, max_value=50.0,
                                    value=0.0,
                                    step=0.1,
                                    help="Enter the percentage change in currency exchange rates."
                                ) / 100  # Convert to decimal

                                st.markdown("### Commodity Price Sensitivity")
                                commodity_price_change = st.number_input(
                                    "Change in Commodity Prices (%)",
                                    min_value=-100.0, max_value=100.0,
                                    value=0.0,
                                    step=0.1,
                                    help="Enter the percentage change in commodity prices."
                                ) / 100  # Convert to decimal

                                for ticker in available_selected:
                                    sensitivity = st.number_input(
                                        f"Sensitivity for {ticker} to Currency and Commodity Changes (%)",
                                        min_value=-10.0, max_value=10.0,
                                        value=0.0,
                                        step=0.1,
                                        key=f"sensitivity_{ticker}"
                                    ) / 100  # Convert to decimal
                                    # Combine currency and commodity changes
                                    adjustment = sensitivity * (currency_rate_change + commodity_price_change)
                                    cross_asset_sensitivity_adjustments[ticker] = adjustment
                            
                            # Submit button for the form
                            submitted = st.form_submit_button("Calculate Risk Metrics")
                        
                        if submitted:
                            with st.spinner("Calculating risk metrics..."):
                                # Calculate portfolio returns as weighted average
                                weights = np.array(portfolio['allocations']) / 100
                                portfolio_returns = returns.copy()
                                
                                # Apply stress shocks if enabled
                                if enable_stress_testing:
                                    for ticker in stress_shocks:
                                        portfolio_returns[ticker] += stress_shocks[ticker]
                                
                                # Apply sensitivity adjustments if enabled
                                if enable_sensitivity_analysis:
                                    for ticker in sensitivity_adjustments:
                                        portfolio_returns[ticker] += sensitivity_adjustments[ticker]
                                
                                # Apply cross-asset sensitivity adjustments if enabled
                                if enable_cross_asset_sensitivity_analysis:
                                    for ticker in cross_asset_sensitivity_adjustments:
                                        portfolio_returns[ticker] += cross_asset_sensitivity_adjustments[ticker]
                                
                                # Ensure no extreme negative returns after adjustments
                                portfolio_returns = portfolio_returns.clip(lower=-1.0)
                                
                                # Calculate weighted portfolio returns
                                portfolio_returns = portfolio_returns.dot(weights)
                                st.session_state.portfolio_returns = portfolio_returns
                                portfolio_skewness = skew(portfolio_returns)
                                portfolio_kurtosis = kurtosis(portfolio_returns)
                                
                                # Calculate VaR and CVaR
                                var = calculate_var(portfolio_returns, confidence_level)
                                cvar = calculate_cvar(portfolio_returns, confidence_level)
                                st.session_state.var = var
                                st.session_state.cvar = cvar
                                
                                # Risk Metrics Summary Table
                                st.markdown("### 📈 Risk Metrics Summary")
                                cum_returns = (1 + portfolio_returns).cumprod()
                                risk_metrics = {
                                    'Value at Risk (VaR)': f"{var:.2f}",
                                    'Conditional Value at Risk (CVaR)': f"{cvar:.2f}",
                                    'Skewness': f"{portfolio_skewness:.2f}",
                                    'Kurtosis': f"{portfolio_kurtosis:.2f}",
                                    'Annualized Volatility (%)': f"{portfolio_returns.std() * 100 * np.sqrt(252):.2f}%",
                                    'Maximum Drawdown (%)': f"{drawdown(cum_returns) * 100:.2f}%"
                                }
                                risk_df = pd.DataFrame(list(risk_metrics.items()), columns=['Metric', 'Value'])
                                st.table(risk_df)
                                st.markdown("""
                                **Tooltip:**
                                - **VaR:** Measures the worst expected loss under normal market conditions over a specific time period at a given confidence level.
                                - **CVaR:** Provides the average loss exceeding the VaR, offering insight into tail risk.
                                - **Annualized Volatility:** Indicates the degree of variation in portfolio returns, representing risk.
                                - **Maximum Drawdown:** Shows the largest peak-to-trough decline, indicating potential risk exposure.
                                """)
                                # Real-Time Risk Warning
                                high_var_threshold = -0.10  # Example: VaR worse than -10%
                                if var <= high_var_threshold:
                                    st.warning(f"⚠️ High VaR! Your portfolio has a {confidence_level*100:.0f}% chance of losing over {abs(var)*100:.2f}% in the next year.")
                                else:
                                    st.success(f"✅ VaR is within acceptable limits: {var*100:.2f}%")

                    with risk_tabs[1]:
                        st.subheader("📊 Visualizations")

                        var = st.session_state.get('var', None)
                        cvar = st.session_state.get('cvar', None)
                        portfolio_returns_local = st.session_state.get('portfolio_returns', None)

                        if var is not None and cvar is not None and portfolio_returns_local is not None:
                            
                            if portfolio_returns_local.empty:
                                st.error("Portfolio returns data is unavailable. Please ensure calculations are correct.")
                                st.stop()
                            
                            # Value at Risk (VaR) Plot (Assuming you want to show cumulative returns over time)
                            st.subheader("Value at Risk (VaR) Plot")
                            fig_var = px.line(
                                portfolio_returns_local.cumsum(),
                                title='Value at Risk (VaR) Over Time',
                                labels={'value': 'Cumulative Returns', 'index': 'Date'}
                            )
                            fig_var.add_hline(y=var, line_dash="dash", line_color="red", annotation_text=f"VaR: {var:.2f}", annotation_position="bottom right")
                            st.plotly_chart(fig_var, use_container_width=True)

                            # Conditional Value at Risk (CVaR) Plot
                            st.subheader("Conditional Value at Risk (CVaR) Plot")
                            fig_cvar = px.line(portfolio_returns_local.cumsum(), title='Conditional Value at Risk (CVaR) Over Time', labels={'value': 'Cumulative Returns', 'index': 'Date'})
                            fig_cvar.add_hline(y=cvar, line_dash="dash", line_color="blue", annotation_text=f"CVaR: {cvar:.2f}", annotation_position="bottom right")
                            st.plotly_chart(fig_cvar, use_container_width=True)

                            # VaR and CVaR Distribution Plot
                            st.subheader("📊 VaR and CVaR Distribution")
                            fig_var_cvar_dist = px.histogram(
                                portfolio_returns_local,
                                nbins=50,
                                title='Returns Distribution with VaR and CVaR',
                                labels={'value': 'Returns', 'count': 'Frequency'}
                            )
                            fig_var_cvar_dist.add_vline(x=var, line_dash="dash", line_color="red", annotation_text=f"VaR: {var:.2f}", annotation_position="top left")
                            fig_var_cvar_dist.add_vline(x=cvar, line_dash="dash", line_color="blue", annotation_text=f"CVaR: {cvar:.2f}", annotation_position="top left")
                            st.plotly_chart(fig_var_cvar_dist, use_container_width=True)
                            st.markdown("""
                            **Interpretation:**
                            - **VaR (Value at Risk):** Indicates the maximum expected loss over a specified period at a given confidence level.
                            - **CVaR (Conditional Value at Risk):** Represents the average loss exceeding the VaR, providing insights into tail risk.
                            - **Annualized Volatility:** Measures the variability of portfolio returns, indicating risk.
                            - **Maximum Drawdown:** Shows the largest peak-to-trough decline, reflecting potential risk exposure.
                            """)
                            
                            st.markdown("""
                            **VaR and CVaR Plots:**
                            - **VaR Plot:** Highlights the point below which a certain percentage of losses fall.
                            - **CVaR Plot:** Shows the average loss in scenarios where losses exceed the VaR.
                            """)
                        else:
                            st.info("Run the Risk Metrics calculation to display VaR and CVaR plots.")
                            
                    with risk_tabs[2]:
                        st.subheader("🛠️ Sensitivity Analysis")
                        # Only plot the Adjusted Returns Distribution if stress testing or sensitivity analysis is enabled
                        if enable_stress_testing or enable_sensitivity_analysis or enable_cross_asset_sensitivity_analysis:
                            st.subheader("📊 Adjusted Returns Distribution")
                            fig_adjusted_returns = px.histogram(portfolio_returns, nbins=50, title='Adjusted Portfolio Returns Distribution',
                                                                labels={'value': 'Returns', 'count': 'Frequency'})
                            st.plotly_chart(fig_adjusted_returns, use_container_width=True)
                            st.markdown("**Interpretation:** This histogram shows how the portfolio returns distribution changes after applying stress shocks and sensitivity adjustments.")

                        # Risk vs Return Scatter Plot
                        st.subheader("📊 Risk vs Return Scatter Plot")
                        risk = returns.std() * np.sqrt(252)
                        ret = returns.mean() * 252

                        if isinstance(risk, (float, int)):
                            scatter_df = pd.DataFrame({
                                'Risk': [risk],
                                'Return': [ret]
                            })
                        else:
                            scatter_df = pd.DataFrame({
                                'Risk': risk,
                                'Return': ret
                            })

                        if isinstance(returns, pd.DataFrame) and returns.shape[1] > 1:
                            scatter_df['Asset'] = available_selected

                        fig_risk_return = px.scatter(scatter_df, x='Risk', y='Return',
                                                     title='Risk vs Return Scatter Plot',
                                                     labels={'Risk': 'Risk (Std Dev)', 'Return': 'Return'},
                                                     hover_data=['Asset'] if isinstance(returns, pd.DataFrame) and returns.shape[1] > 1 else [])
                        st.plotly_chart(fig_risk_return, use_container_width=True)

    else:
        st.warning("Please add at least one portfolio to run risk analysis.")