for the consistency, please delete the portfolio start/end date and the portfolio benchmark, and everything will change to only use the backtest date and backtest benchmark(the one in Select Portfolios to Compare)

New Portfolio Name
Retirement Portfolio_Copy
Confirm Duplicate
refine the interface/place of New Portfolio Name, its too ugly right now

 refine the Visualizations tab as it is messy up right now

Please review my code and provide the original code alongside the updated one. Include the exact instructions for where and how to apply the corrections in the original code by referencing specific function names, variable names, or line numbers if possible. Dont give me general corrections as I am a newbie in coding, so make your response concise and simple, showing only the revised code snippets with a brief explanation for each change.

%%writefile portfolio_optimizer.py
import streamlit as st
import pandas as pd
import plotly.graph_objects as go
import numpy as np
import yfinance as yf
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
import plotly.express as px
import warnings
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from scipy.stats import skew, kurtosis
from joblib import Parallel, delayed
import json
from functools import lru_cache
from concurrent.futures import ProcessPoolExecutor
from scipy.optimize import minimize
from scipy.stats import norm

warnings.filterwarnings('ignore')

# ----------------------------
# Frequency Mapping
# ----------------------------
frequency_mapping = {
    "Daily": "D",
    "Weekly": "W",
    "Monthly": "M",
    "Quarterly": "Q",
    "Yearly": "Y"
}

# ----------------------------
# Helper Functions
# ----------------------------

def get_company_name(ticker_df, ticker):
    match = ticker_df[ticker_df['Ticker'] == ticker]
    if not match.empty:
        return match.iloc[0]['Company Name']
    return "Unknown"

def format_asset_option(ticker, company_name):
    return f"{ticker} - {company_name}"

def calculate_final_score(primary_metrics, comparison_metrics):
    score = 0
    total = 0
    metrics_to_compare = [
        'Annualized Return (CAGR)',
        'Sharpe Ratio',
        'Sortino Ratio',
        'Treynor Ratio',
        'Calmar Ratio',
        'Alpha (annualized)',
        'Information Ratio',
        'Modigliani–Modigliani Measure',
        'Upside Capture Ratio',
        'Gain/Loss Ratio'
    ]
    metric_weights = {
        'Annualized Return (CAGR)': 15,
        'Sharpe Ratio': 15,
        'Sortino Ratio': 10,
        'Treynor Ratio': 10,
        'Calmar Ratio': 10,
        'Alpha (annualized)': 10,
        'Information Ratio': 10,
        'Modigliani–Modigliani Measure': 10,
        'Upside Capture Ratio': 5,
        'Gain/Loss Ratio': 5
    }

    for metric in metrics_to_compare:
        primary = primary_metrics.get(metric, "N/A")
        comparison = comparison_metrics.get(metric, "N/A")
        if primary != "N/A" and comparison != "N/A":
            try:
                primary_val = float(primary.strip('%')) if '%' in primary else float(primary)
                comparison_val = float(comparison.strip('%')) if '%' in comparison else float(comparison)
                weight = metric_weights.get(metric, 1)
                if comparison_val == 0:
                    st.warning(f"Comparison metric '{metric}' has a value of zero. Skipping this metric to avoid division by zero.")
                    continue
                if primary_val > comparison_val:
                    score += weight * (primary_val / comparison_val)
                else:
                    score += weight * (primary_val / comparison_val) * 0.5  # Partial credit
                total += weight
            except:
                continue
    return (score / total) * 100 if total > 0 else 0

def plot_growth_comparison(cum_returns, benchmark_cum_returns):
    try:
        df = pd.DataFrame({
            'Date': cum_returns.index,
            'Portfolio': cum_returns.values,
            'Benchmark': benchmark_cum_returns.values
        }).reset_index(drop=True)  # Ensure 'Date' is a column
        fig = px.line(
            df, 
            x='Date', 
            y=['Portfolio', 'Benchmark'], 
            title='Growth Comparison', 
            labels={'value': 'Cumulative Returns', 'Date': 'Date'}, 
            hover_data=['Date']
        )
        fig.update_layout(
            hovermode='x unified',
            xaxis=dict(rangeslider=dict(visible=True), type='date'),
            plot_bgcolor='rgba(0,0,0,0)',       # Transparent background
            paper_bgcolor='rgba(0,0,0,0)',      # Transparent paper
            font=dict(color='white')             # White font for dark backgrounds
        )
        fig.update_traces(line=dict(width=2))
        st.plotly_chart(fig, use_container_width=True, config={
            'responsive': True,
            'scrollZoom': True,
            'displayModeBar': True
        })
    except Exception as e:
        st.error(f"Error plotting growth comparison: {e}")

def plot_drawdown_comparison(drawdown_portfolio, drawdown_benchmark):
    try:
        df = pd.DataFrame({
            'Portfolio Drawdown': drawdown_portfolio,
            'Benchmark Drawdown': drawdown_benchmark
        })
        fig = px.line(df, title='Drawdown Comparison', labels={'value': 'Drawdown (%)', 'index': 'Date'})
        fig.update_layout(
            hovermode='x unified',
            xaxis=dict(rangeslider=dict(visible=True), type='date'),
            plot_bgcolor='rgba(0,0,0,0)',
            paper_bgcolor='rgba(0,0,0,0)',
            font=dict(color='white')
        )
        fig.update_traces(line=dict(width=2))
        st.plotly_chart(fig, use_container_width=True, config={
            'responsive': True,
            'scrollZoom': True,
            'displayModeBar': True,
            'modeBarButtonsToRemove': ['zoom2d', 'pan2d']  # Remove duplicate or unwanted buttons
        })
    except Exception as e:
        st.error(f"Error plotting drawdown comparison: {e}")

def backtest(weights, prices, rebalance_freq='M', broker_fee=0.0, debug=False):
    try:
        returns = prices.pct_change().dropna()
        if returns.empty:
            st.error("Returns data is empty after calculating percentage changes.")
            return pd.Series(dtype=float), pd.Series(dtype=float)
        portfolio_returns = returns.dot(weights)
        if not isinstance(portfolio_returns, pd.Series):
            portfolio_returns = portfolio_returns.squeeze()

        # Identify rebalancing dates
        if rebalance_freq == 'D':
            rebalance_dates = returns.index
        else:
            rebalance_dates = returns.resample(rebalance_freq).last().dropna().index

        # Debug: Show rebalancing dates
        if debug:
            st.write(f"Rebalancing Dates: {rebalance_dates.tolist()}")

        # Ensure rebalance_dates are in the portfolio_returns index
        valid_rebalance_dates = rebalance_dates.intersection(portfolio_returns.index)
        if debug:
            st.write(f"Valid Rebalancing Dates: {valid_rebalance_dates.tolist()}")

        # Apply broker fees on rebalancing dates
        if not valid_rebalance_dates.empty:
            portfolio_returns.loc[valid_rebalance_dates] -= broker_fee / 100  # Convert to decimal
        else:
            st.warning("No valid rebalancing dates found within the returns data.")

        cum_returns = (1 + portfolio_returns).cumprod()
        if isinstance(cum_returns, pd.DataFrame):
            if cum_returns.shape[1] == 1:
                cum_returns = cum_returns.iloc[:,0]
            else:
                st.error("Cumulative returns have multiple columns. Cannot proceed.")
                return pd.Series(dtype=float), pd.Series(dtype=float)

        # Debug: Show cumulative returns
        if debug:
            st.write("Cumulative Returns:")
            st.write(cum_returns)

        return portfolio_returns, cum_returns
    except Exception as e:
        st.error(f"Error during backtesting: {e}")
        return pd.Series(dtype=float), pd.Series(dtype=float)

def calculate_simple_metrics(returns, cum_returns, start_balance=10000):
    metrics = {}
    metrics['Start Balance'] = f"${start_balance:,.2f}"
    if not cum_returns.empty:
        try:
            end_balance = start_balance * cum_returns.iloc[-1]
            metrics['End Balance'] = f"${end_balance:,.2f}"

            start_date = cum_returns.index[0]
            end_date = cum_returns.index[-1]
            days = (end_date - start_date).days
            years = days / 365.25
            cagr = (end_balance / start_balance) ** (1 / years) - 1 if years > 0 else np.nan
            metrics['Annualized Return (CAGR)'] = f"{cagr * 100:.2f}%" if not np.isnan(cagr) else "N/A"

            yearly_returns = returns.resample('Y').apply(lambda x: (1 + x).prod() - 1)
            best_year = yearly_returns.max() * 100 if not yearly_returns.empty else np.nan
            worst_year = yearly_returns.min() * 100 if not yearly_returns.empty else np.nan
            metrics['Best Year'] = f"{best_year:.2f}%" if not np.isnan(best_year) else "N/A"
            metrics['Worst Year'] = f"{worst_year:.2f}%" if not np.isnan(worst_year) else "N/A"

            metrics['Arithmetic Mean (Monthly)'] = f"{returns.mean() * 100:.2f}%"
            metrics['Arithmetic Mean (Annualized)'] = f"{returns.mean() * 252 * 100:.2f}%"
            metrics['Geometric Mean (Monthly)'] = f"{(np.exp(np.log1p(returns).mean()) - 1) * 100:.2f}%"
            metrics['Geometric Mean (Annualized)'] = f"{(np.exp(np.log1p(returns).mean() * 252) - 1) * 100:.2f}%"
            metrics['Standard Deviation (Monthly)'] = f"{returns.std() * 100:.2f}%"
            metrics['Standard Deviation (Annualized)'] = f"{returns.std() * np.sqrt(252) * 100:.2f}%"
            metrics['Downside Deviation (Monthly)'] = f"{returns[returns < 0].std() * 100:.2f}%"
            metrics['Maximum Drawdown'] = f"{drawdown(cum_returns) * 100:.2f}%" if not cum_returns.empty else "N/A"
            metrics['Sharpe Ratio'] = f"{calculate_sharpe_ratio(returns, 0.02):.2f}" if not cum_returns.empty else "N/A"
            metrics['Sortino Ratio'] = f"{calculate_sortino_ratio(returns, 0.02):.2f}" if not cum_returns.empty else "N/A"
            metrics['Gain/Loss Ratio'] = f"{calculate_gain_loss_ratio(returns):.2f}" if not cum_returns.empty else "N/A"
            metrics['Skewness'] = f"{skew(returns):.2f}" if not returns.empty else "N/A"
            metrics['Excess Kurtosis'] = f"{kurtosis(returns):.2f}" if not returns.empty else "N/A"
            metrics['Safe Withdrawal Rate'] = f"{calculate_safe_withdrawal_rate(returns):.6f}%" if not returns.empty else "N/A"
            metrics['Perpetual Withdrawal Rate'] = f"{calculate_perpetual_withdrawal_rate(returns):.6f}%" if not returns.empty else "N/A"
            metrics['Positive Periods'] = calculate_positive_periods(returns) if not returns.empty else "N/A"
        except Exception as e:
            st.error(f"Error calculating simple metrics: {e}")
            for key in ['End Balance', 'Annualized Return (CAGR)', 'Best Year', 'Worst Year',
                        'Arithmetic Mean (Monthly)', 'Arithmetic Mean (Annualized)',
                        'Geometric Mean (Monthly)', 'Geometric Mean (Annualized)',
                        'Standard Deviation (Monthly)', 'Standard Deviation (Annualized)',
                        'Downside Deviation (Monthly)', 'Maximum Drawdown',
                        'Sharpe Ratio', 'Sortino Ratio', 'Gain/Loss Ratio',
                        'Skewness', 'Excess Kurtosis', 'Safe Withdrawal Rate',
                        'Perpetual Withdrawal Rate', 'Positive Periods']:
                metrics[key] = "N/A"
    else:
        metrics['End Balance'] = "N/A"
        metrics['Annualized Return (CAGR)'] = "N/A"
        metrics['Best Year'] = "N/A"
        metrics['Worst Year'] = "N/A"
        metrics['Arithmetic Mean (Monthly)'] = "N/A"
        metrics['Arithmetic Mean (Annualized)'] = "N/A"
        metrics['Geometric Mean (Monthly)'] = "N/A"
        metrics['Geometric Mean (Annualized)'] = "N/A"
        metrics['Standard Deviation (Monthly)'] = "N/A"
        metrics['Standard Deviation (Annualized)'] = "N/A"
        metrics['Downside Deviation (Monthly)'] = "N/A"
        metrics['Maximum Drawdown'] = "N/A"
        metrics['Sharpe Ratio'] = "N/A"
        metrics['Sortino Ratio'] = "N/A"
        metrics['Gain/Loss Ratio'] = "N/A"
        metrics['Skewness'] = "N/A"
        metrics['Excess Kurtosis'] = "N/A"
        metrics['Safe Withdrawal Rate'] = "N/A"
        metrics['Perpetual Withdrawal Rate'] = "N/A"
        metrics['Positive Periods'] = "N/A"

    return metrics

def calculate_sharpe_ratio(returns, rf=0.02):
    """
    Calculate the Sharpe Ratio for a given set of returns.

    Parameters:
    - returns (pd.Series): Daily returns of the portfolio.
    - rf (float): Risk-free rate (default is 2%).

    Returns:
    - float: Sharpe Ratio.
    """
    excess_return = returns.mean() * 252 - rf
    std_dev = returns.std() * np.sqrt(252)
    return excess_return / std_dev if std_dev != 0 else np.nan

def calculate_sortino_ratio(returns, rf=0.02):
    excess_return = returns.mean() * 252 - rf
    downside_std = returns[returns < 0].std() * np.sqrt(252)
    return excess_return / downside_std if downside_std != 0 else np.nan

def calculate_treynor_ratio(returns, benchmark_returns, rf=0.02):
    beta = calculate_beta(returns, benchmark_returns)
    excess_return = returns.mean() * 252 - rf
    return excess_return / beta if beta != 0 else np.nan

def calculate_calmar_ratio(returns, cum_returns):
    annual_return = returns.mean() * 252
    max_dd = drawdown(cum_returns)
    return annual_return / abs(max_dd) if max_dd != 0 else np.nan

def calculate_beta(returns, benchmark_returns):
    if returns.empty or benchmark_returns.empty:
        return np.nan
    covariance = np.cov(returns, benchmark_returns)
    return covariance[0,1] / covariance[1,1] if covariance[1,1] != 0 else np.nan

def calculate_alpha(returns, benchmark_returns, rf=0.02):
    beta = calculate_beta(returns, benchmark_returns)
    portfolio_return = returns.mean() * 252
    benchmark_return = benchmark_returns.mean() * 252
    return portfolio_return - (rf + beta * (benchmark_return - rf)) if not np.isnan(beta) else np.nan

def calculate_r_squared(returns, benchmark_returns):
    if returns.empty or benchmark_returns.empty:
        return np.nan
    covariance = np.cov(returns, benchmark_returns)
    var_port = covariance[0,0]
    var_bench = covariance[1,1]
    cov = covariance[0,1]
    return (cov ** 2) / (var_port * var_bench) if var_port !=0 and var_bench !=0 else np.nan

def calculate_information_ratio(returns, benchmark_returns):
    if returns.empty or benchmark_returns.empty:
        return np.nan
    active_return = (returns.mean() - benchmark_returns.mean()) * 252
    tracking_error = calculate_tracking_error(returns, benchmark_returns)
    return active_return / tracking_error if tracking_error != 0 else np.nan
    
def get_common_benchmarks(selected_tickers):
    # Define benchmark suggestions based on asset sectors or indices
    sp500 = {"AAPL", "MSFT", "GOOGL", "AMZN", "FB", "TSLA"}  # Example S&P 500 tech companies
    nasdaq_tech = {"AAPL", "MSFT", "GOOGL", "AMZN", "FB", "TSLA", "NVDA", "AMD"}
    
    selected_set = set(selected_tickers)
    
    if selected_set.issubset(sp500):
        return {
            "S&P 500": "^GSPC",
            "Dow Jones Industrial Average": "^DJI",
            "Russell 2000": "^RUT",
            "Custom": "CUSTOM"
        }
    elif selected_set.issubset(nasdaq_tech):
        return {
            "NASDAQ Composite": "^IXIC",
            "QQQ (Invesco QQQ ETF)": "QQQ",
            "Custom": "CUSTOM"
        }
    else:
        return {
            "S&P 500": "^GSPC",
            "NASDAQ Composite": "^IXIC",
            "Dow Jones Industrial Average": "^DJI",
            "Russell 2000": "^RUT",
            "Custom": "CUSTOM"
        }

def calculate_tracking_error(returns, benchmark_returns):
    if returns.empty or benchmark_returns.empty:
        return np.nan
    return np.std((returns - benchmark_returns)) * np.sqrt(252)

def calculate_active_return(returns, benchmark_returns):
    if returns.empty or benchmark_returns.empty:
        return np.nan
    return (returns.mean() - benchmark_returns.mean()) * 252 * 100

def calculate_gain_loss_ratio(returns):
    gains = returns[returns > 0].sum()
    losses = -returns[returns < 0].sum()
    return gains / losses if losses != 0 else np.nan

def drawdown(cum_returns):
    if cum_returns.empty:
        return np.nan
    peak = cum_returns.expanding(min_periods=1).max()
    dd = (cum_returns / peak) - 1
    return dd.min()

def calculate_capture_ratio(returns, benchmark_returns, upside=True):
    if returns.empty or benchmark_returns.empty:
        return np.nan
    mask = benchmark_returns > 0 if upside else benchmark_returns < 0
    if mask.sum() == 0:
        return np.nan
    portfolio = returns[mask]
    benchmark = benchmark_returns[mask]
    return (portfolio.sum() / benchmark.sum()) * 100 if benchmark.sum() != 0 else np.nan

def calculate_safe_withdrawal_rate(returns):
    if returns.empty:
        return np.nan
    return (returns.mean() / returns.std()) * 100

def calculate_perpetual_withdrawal_rate(returns):
    if returns.empty:
        return np.nan
    return (returns.mean() / returns.std()) * 100

def calculate_positive_periods(returns):
    if returns.empty:
        return "N/A"
    positive = returns > 0
    return f"{positive.sum()} out of {len(returns)} ({(positive.sum()/len(returns))*100:.2f}%)"

def calculate_modigliani_miller(returns, benchmark_returns, rf=0.02):
    sharpe = calculate_sharpe_ratio(returns, rf)
    alpha = calculate_alpha(returns, benchmark_returns, rf)
    return alpha / sharpe if sharpe != 0 and not np.isnan(alpha) else np.nan

def get_drawdown_details(cum_returns):
    drawdowns = []
    if cum_returns.empty:
        return drawdowns
    peak = cum_returns.iloc[0]
    peak_date = cum_returns.index[0]
    trough = cum_returns.iloc[0]
    trough_date = cum_returns.index[0]
    for date, value in cum_returns.items():
        if value > peak:
            if trough < peak:
                recovery = cum_returns[cum_returns >= peak].loc[trough_date:]
                if not recovery.empty:
                    recovery_date = recovery.index[0]
                    if recovery_date > trough_date:
                        recovery_time = (recovery_date - trough_date).days
                        underwater_period = (recovery_date - peak_date).days
                        drawdowns.append({
                            'Start': peak_date.strftime('%b %Y'),
                            'End': trough_date.strftime('%b %Y'),
                            'Length': f"{(trough_date - peak_date).days // 30} months",
                            'Recovery By': recovery_date.strftime('%b %Y'),
                            'Recovery Time': f"{recovery_time // 30} months",
                            'Underwater Period': f"{underwater_period // 30} months",
                            'Drawdown': f"{((trough / peak) - 1) * 100:.2f}%"
                        })
                    else:
                        drawdowns.append({
                            'Start': peak_date.strftime('%b %Y'),
                            'End': trough_date.strftime('%b %Y'),
                            'Length': f"{(trough_date - peak_date).days // 30} months",
                            'Recovery By': "Not Recovered",
                            'Recovery Time': "N/A",
                            'Underwater Period': "N/A",
                            'Drawdown': f"{((trough / peak) - 1) * 100:.2f}%"
                        })
            peak = value
            peak_date = date
            trough = value
            trough_date = date
        elif value < trough:
            trough = value
            trough_date = date
    if trough < peak:
        drawdowns.append({
            'Start': peak_date.strftime('%b %Y'),
            'End': trough_date.strftime('%b %Y'),
            'Length': f"{(trough_date - peak_date).days // 30} months",
            'Recovery By': "Not Recovered",
            'Recovery Time': "N/A",
            'Underwater Period': "N/A",
            'Drawdown': f"{((trough / peak) - 1) * 100:.2f}%"
        })
    try:
        drawdowns_sorted = sorted(drawdowns, key=lambda x: float(x['Drawdown'].strip('%')), reverse=False)
    except:
        drawdowns_sorted = []
    drawdowns_sorted = drawdowns_sorted[:10]
    for idx, dd in enumerate(drawdowns_sorted, start=1):
        dd['Rank'] = idx
    drawdowns_final = []
    for dd in drawdowns_sorted:
        drawdowns_final.append({
            'Rank': dd['Rank'],
            'Start': dd['Start'],
            'End': dd['End'],
            'Length': dd['Length'],
            'Recovery By': dd['Recovery By'],
            'Recovery Time': dd['Recovery Time'],
            'Underwater Period': dd['Underwater Period'],
            'Drawdown': dd['Drawdown']
        })
    return drawdowns_final

def optimize_portfolio(returns, objectives=['sharpe'], rf=0.02, max_weight=1.0, min_weight=0.0, target_return=None):
    def sharpe_ratio(weights):
        portfolio_return = np.dot(returns.mean(), weights) * 252
        portfolio_volatility = np.sqrt(np.dot(weights.T, np.dot(returns.cov() * 252, weights)))
        return -portfolio_return / portfolio_volatility if portfolio_volatility > 1e-6 else 0  # Negative for maximization
    
    def min_variance(weights):
        return np.dot(weights.T, np.dot(returns.cov() * 252, weights))
    
    def max_return(weights):
        return -np.dot(returns.mean(), weights) * 252  # Negative for maximization
    
    def min_drawdown(weights):
        # Simplistic drawdown minimization using expected shortfall
        portfolio_returns = returns.dot(weights)
        var = calculate_var(portfolio_returns, 0.95)
        cvar = calculate_cvar(portfolio_returns, 0.95)
        return cvar  # Minimizing CVaR as a proxy for drawdown
    
    # Define objective functions based on selected objectives
    objective_functions = []
    for obj in objectives:
        if obj == 'sharpe':
            objective_functions.append(sharpe_ratio)
        elif obj == 'min_variance':
            objective_functions.append(min_variance)
        elif obj == 'max_return':
            objective_functions.append(max_return)
        elif obj == 'min_drawdown':
            objective_functions.append(min_drawdown)
        else:
            st.error(f"Invalid optimization objective: {obj}")
            return None
    
    # Composite objective function: weighted sum of individual objectives
    def composite_objective(weights):
        return sum(fn(weights) for fn in objective_functions)
    
    constraints = [
        {'type': 'eq', 'fun': lambda x: np.sum(x) - 1},
    ]
    
    if target_return is not None:
        constraints.append({
            'type': 'ineq',
            'fun': lambda x: np.dot(x, returns.mean()) * 252 - target_return
        })
    
    bounds = tuple((min_weight, max_weight) for asset in range(returns.shape[1]))
    initial_guess = [1.0 / returns.shape[1]] * returns.shape[1]
    
    result = minimize(composite_objective, initial_guess, method='SLSQP', bounds=bounds, constraints=constraints)
    if result.success:
        portfolio_volatility = np.sqrt(np.dot(result.x.T, np.dot(returns.cov() * 252, result.x)))
        if portfolio_volatility <= 1e-6:
            st.error("Optimized portfolio has near-zero volatility. Optimization constraints may be too restrictive.")
            return None
        return result.x
    else:
        st.error("Optimization failed. Try adjusting your constraints or target return.")
        return None

def monte_carlo_simulation(returns, num_simulations=1000, periods=252, mean_returns=None, cov_matrix=None):
    if mean_returns is None:
        mean_returns = returns.mean()
    if cov_matrix is None:
        cov_matrix = returns.cov()
    
    num_assets = returns.shape[1]
    
    def simulate():
        weights = np.random.random(num_assets)
        weights /= np.sum(weights)
        simulated_daily_returns = np.random.multivariate_normal(mean_returns, cov_matrix, periods)
        simulated_portfolio_returns = np.sum(simulated_daily_returns * weights, axis=1)
        cumulative_return = np.prod(1 + simulated_portfolio_returns) - 1  # Annual return
        return cumulative_return
    
    portfolio_returns = Parallel(n_jobs=-1)(
        delayed(simulate)() for _ in range(num_simulations)
    )

    return np.array(portfolio_returns)
    
    try:
        num_assets = returns.shape[1] if returns.ndim > 1 else 1
        results = Parallel(n_jobs=-1)(
            delayed(generate_portfolio)(returns, rf) for _ in range(num_portfolios)
        )
        
        # Filter out incomplete results (where any element is NaN)
        valid_results = [res for res in results if all(not np.isnan(x) for x in res)]
        
        if len(valid_results) == 0:
            st.warning("No valid portfolios to plot on the Efficient Frontier.")
            return
        
        results = np.array(valid_results).T
        ef_df = pd.DataFrame({
            'Std Dev': results[0],
            'Return': results[1],
            'Sharpe Ratio': results[2]
        })
        fig = px.scatter(
            ef_df,
            x='Std Dev',
            y='Return',
            color='Sharpe Ratio',
            color_continuous_scale='Viridis',
            title='Efficient Frontier',
            labels={
                'Std Dev': 'Annualized Volatility (Std Dev)',
                'Return': 'Annualized Return',
                'Sharpe Ratio': 'Sharpe Ratio'
            },
            hover_data={
                'Sharpe Ratio': ':.2f',
                'Std Dev': ':.2f',
                'Return': ':.2f'
            },
            template='plotly_white'
        )
        
        # Add annotation for the maximum Sharpe ratio portfolio
        max_sharpe_idx = ef_df['Sharpe Ratio'].idxmax()
        max_sharpe = ef_df.loc[max_sharpe_idx]
        fig.add_annotation(
            x=max_sharpe['Std Dev'],
            y=max_sharpe['Return'],
            text="Max Sharpe",
            showarrow=True,
            arrowhead=1
        )
        
        # Check for Optimized Portfolio in Session State
        optimized_weights = st.session_state.backtest_results.get('optimized_weights', None)
        optimized_return = st.session_state.backtest_results.get('optimized_return', None)
        optimized_volatility = st.session_state.backtest_results.get('optimized_volatility', None)
        optimized_sharpe = st.session_state.backtest_results.get('optimized_sharpe', None)
        
        if optimized_weights is not None and optimized_return is not None and optimized_volatility is not None and optimized_sharpe is not None:
            if optimized_volatility > 1e-6 and np.isfinite(optimized_sharpe):
                # Add optimized portfolio to the existing Efficient Frontier plot
                fig.add_trace(go.Scatter(
                    x=[optimized_volatility],
                    y=[optimized_return],
                    mode='markers+text',
                    marker=dict(color='red', size=14, symbol='star'),
                    name='Optimized Portfolio',
                    text=["Optimized"],
                    textposition="top center",
                    hoverinfo='text'
                ))
                fig.add_annotation(
                    x=optimized_volatility,
                    y=optimized_return,
                    text="Optimized",
                    showarrow=True,
                    arrowhead=2,
                    ax=0,
                    ay=-40
                )
        
        st.plotly_chart(fig, use_container_width=True, config={
            'responsive': True,
            'scrollZoom': True,
            'displayModeBar': True
        })
    except Exception as e:
        st.error(f"Error plotting efficient frontier: {e}")

def generate_portfolio(returns, rf):
    try:
        weights = np.random.random(returns.shape[1])
        weights /= np.sum(weights)
        portfolio_return = np.dot(weights, returns.mean()) * 252
        portfolio_volatility = np.sqrt(np.dot(weights.T, np.dot(returns.cov() * 252, weights)))
        sharpe_ratio = (portfolio_return - rf) / portfolio_volatility if portfolio_volatility > 0 else 0
        return portfolio_volatility, portfolio_return, sharpe_ratio
    except Exception as e:
        st.error(f"Error generating portfolio: {e}")
        return np.nan, np.nan, np.nan  # Return NaNs to indicate failure
        
def plot_allocation_pie(weights, assets, title='Portfolio Allocation'):
    allocation_df = pd.DataFrame({
        'Asset': assets,
        'Weight': weights
    })
    fig = px.pie(
        allocation_df, 
        names='Asset', 
        values='Weight', 
        title=title,
        color='Asset',
        color_discrete_sequence=px.colors.qualitative.Set3  # Updated color sequence
    )
    st.plotly_chart(fig, use_container_width=True)

def plot_rolling_return_comparison(cum_returns, benchmark_cum_returns, window=252):
    try:
        rolling_return = cum_returns.pct_change(window).dropna() * 252  # Annualized rolling return
        benchmark_rolling_return = benchmark_cum_returns.pct_change(window).dropna() * 252

        df = pd.DataFrame({
            'Date': rolling_return.index,
            'Portfolio': rolling_return.values,
            'Benchmark': benchmark_rolling_return.values
        }).reset_index(drop=True)  # Ensure 'Date' is a column

        df_melted = df.melt(id_vars='Date', value_vars=['Portfolio', 'Benchmark'], var_name='Series', value_name='Annualized Rolling Return')

        fig = px.line(
            df_melted, 
            x='Date', 
            y='Annualized Rolling Return', 
            color='Series',
            title=f'Rolling {window}-Day Annualized Return Comparison',
            labels={'Annualized Rolling Return': 'Annualized Rolling Return', 'Date': 'Date'}
        )
        fig.update_layout(
            hovermode='x unified',
            xaxis=dict(rangeslider=dict(visible=True), type='date'),
            template='plotly_dark'
        )
        fig.update_traces(line=dict(width=2))
        st.plotly_chart(fig, use_container_width=True, config={
            'responsive': True,
            'scrollZoom': True,
            'displayModeBar': True
        })
    except Exception as e:
        st.error(f"Error plotting rolling return comparison: {e}")
        
def plot_correlation_heatmap(returns):
    try:
        if isinstance(returns, pd.DataFrame) and returns.shape[1] >= 2:
            valid_columns = [col for col in returns.columns if returns[col].nunique() > 1 and returns[col].notna().sum() >= 10]
            if len(valid_columns) < 2:
                st.warning("Correlation heatmap requires at least two assets with variability and sufficient data.")
                return
            corr_matrix = returns[valid_columns].corr()
            fig = px.imshow(
                corr_matrix, 
                title='Asset Correlation Heatmap',
                labels={'x': 'Asset', 'y': 'Asset', 'color': 'Correlation'},
                color_continuous_scale='Portland',  # Updated color scale
                zmin=-1, 
                zmax=1,
                text_auto=True,
                aspect="auto",
                template='plotly_dark'               # Dark theme for better contrast
            )
            fig.update_xaxes(side="top")  # Move x-axis labels to the top for better readability
            st.plotly_chart(fig, use_container_width=True)
        elif isinstance(returns, pd.Series) and returns.nunique() > 1:
            st.warning("Correlation heatmap requires at least two assets with variability.")
        else:
            st.warning("Correlation heatmap requires at least two assets with variability.")
    except Exception as e:
        st.error(f"Error plotting correlation heatmap: {e}")

def plot_rolling_volatility(returns, window=252):
    rolling_vol = returns.rolling(window).std() * np.sqrt(252)
    fig = px.line(rolling_vol, title=f'Rolling {window}-Day Volatility', labels={'value': 'Volatility', 'index': 'Date'})
    st.plotly_chart(fig, use_container_width=True)

def plot_var_cvar_distribution(portfolio_returns, var, cvar):
    fig = px.histogram(portfolio_returns, nbins=50, title='Returns Distribution with VaR and CVaR',
                       labels={'value': 'Returns', 'count': 'Frequency'})
    fig.add_vline(x=var, line_dash="dash", line_color="red", annotation_text=f"VaR: {var:.2f}", annotation_position="top left")
    fig.add_vline(x=cvar, line_dash="dash", line_color="blue", annotation_text=f"CVaR: {cvar:.2f}", annotation_position="top left")
    st.plotly_chart(fig, use_container_width=True)

def plot_var_cvar_over_time(portfolio_returns, var, cvar):
    cumulative_returns = (1 + portfolio_returns).cumprod()
    fig = px.line(cumulative_returns, title='Cumulative Returns with VaR and CVaR Over Time', labels={'value': 'Cumulative Returns', 'index': 'Date'})
    fig.add_hline(y=var, line_dash="dash", line_color="red", annotation_text=f"VaR: {var:.2f}", annotation_position="bottom right")
    fig.add_hline(y=cvar, line_dash="dash", line_color="blue", annotation_text=f"CVaR: {cvar:.2f}", annotation_position="bottom right")
    st.plotly_chart(fig, use_container_width=True)

def calculate_var(returns, confidence_level=0.95):
    return np.percentile(returns, 100 * (1 - confidence_level))

def calculate_cvar(returns, confidence_level=0.95):
    var = calculate_var(returns, confidence_level)
    return returns[returns <= var].mean()

# ----------------------------
# Caching Functions
# ----------------------------
@st.cache_data(show_spinner=False)
def calculate_portfolio_metrics(returns, cum_returns, rf, benchmark_returns=None):
    metrics = calculate_simple_metrics(returns, cum_returns)

    if benchmark_returns is not None and not benchmark_returns.empty:
        correlation = returns.corr(benchmark_returns)
        beta = calculate_beta(returns, benchmark_returns)
        metrics['Benchmark Correlation'] = f"{correlation:.2f}" if not np.isnan(correlation) else "N/A"
        metrics['Beta'] = f"{beta:.2f}" if not np.isnan(beta) else "N/A"

        alpha = calculate_alpha(returns, benchmark_returns, rf)
        metrics['Alpha (annualized)'] = f"{alpha * 100:.2f}%" if not np.isnan(alpha) else "N/A"

        r2 = calculate_r_squared(returns, benchmark_returns)
        metrics['R2'] = f"{r2 * 100:.2f}%" if not np.isnan(r2) else "N/A"

        treynor = calculate_treynor_ratio(returns, benchmark_returns, rf)
        metrics['Treynor Ratio'] = f"{treynor:.2f}" if not np.isnan(treynor) else "N/A"

        calmar = calculate_calmar_ratio(returns, cum_returns)
        metrics['Calmar Ratio'] = f"{calmar:.2f}" if not np.isnan(calmar) else "N/A"

        m2 = calculate_modigliani_miller(returns, benchmark_returns, rf)
        metrics['Modigliani–Modigliani Measure'] = f"{m2 * 100:.2f}%" if not np.isnan(m2) else "N/A"

        info_ratio = calculate_information_ratio(returns, benchmark_returns)
        metrics['Information Ratio'] = f"{info_ratio:.2f}" if not np.isnan(info_ratio) else "N/A"

        tracking_error = calculate_tracking_error(returns, benchmark_returns)
        metrics['Tracking Error'] = f"{tracking_error * 100:.2f}%" if not np.isnan(tracking_error) else "N/A"

        active_return = calculate_active_return(returns, benchmark_returns)
        metrics['Active Return'] = f"{active_return:.2f}%" if not np.isnan(active_return) else "N/A"

        upside_capture = calculate_capture_ratio(returns, benchmark_returns, upside=True)
        downside_capture = calculate_capture_ratio(returns, benchmark_returns, upside=False)
        metrics['Upside Capture Ratio'] = f"{upside_capture:.2f}%" if not np.isnan(upside_capture) else "N/A"
        metrics['Downside Capture Ratio'] = f"{downside_capture:.2f}%" if not np.isnan(downside_capture) else "N/A"
    else:
        metrics.update({
            'Benchmark Correlation': "N/A",
            'Beta': "N/A",
            'Alpha (annualized)': "N/A",
            'R2': "N/A",
            'Treynor Ratio': "N/A",
            'Calmar Ratio': "N/A",
            'Modigliani–Modigliani Measure': "N/A",
            'Information Ratio': "N/A",
            'Tracking Error': "N/A",
            'Active Return': "N/A",
            'Upside Capture Ratio': "N/A",
            'Downside Capture Ratio': "N/A"
        })

    return metrics
    
@st.cache_data(show_spinner=False)
def get_tickers():
    try:
        # Fetch S&P 500 companies
        sp500_table = pd.read_html('https://en.wikipedia.org/wiki/List_of_S%26P_500_companies')[0]
        sp500 = sp500_table[['Symbol', 'Security']].rename(columns={'Symbol': 'Ticker', 'Security': 'Company Name'})
        sp500['Ticker'] = sp500['Ticker'].str.replace('.', '-', regex=False)
        
        # Fetch NASDAQ-100 companies
        nasdaq100_tables = pd.read_html('https://en.wikipedia.org/wiki/NASDAQ-100')
        nasdaq100 = pd.DataFrame()
        for table in nasdaq100_tables:
            if 'Ticker' in table.columns and 'Company' in table.columns:
                nasdaq100 = table[['Ticker', 'Company']].rename(columns={'Ticker': 'Ticker', 'Company': 'Company Name'})
                nasdaq100['Ticker'] = nasdaq100['Ticker'].str.replace('.', '-', regex=False)
                break
        
        combined = pd.concat([sp500, nasdaq100], ignore_index=True)
        combined = combined.drop_duplicates(subset=['Ticker'])
        return combined.sort_values('Ticker').reset_index(drop=True)
    except Exception as e:
        st.error(f"Error fetching tickers: {e}")
        return pd.DataFrame(columns=['Ticker', 'Company Name'])

@st.cache_data(show_spinner=False)
def download_data(tickers, start, end, retries=3, backoff_factor=0.3):
    try:
        session = requests.Session()
        retry = Retry(
            total=retries,
            read=retries,
            connect=retries,
            backoff_factor=backoff_factor,
            status_forcelist=(500, 502, 504),
        )
        adapter = HTTPAdapter(max_retries=retry)
        session.mount('http://', adapter)
        session.mount('https://', adapter)
        data = yf.download(tickers, start=start, end=end, progress=False, session=session)['Adj Close']
        if isinstance(data, pd.Series):
            data = data.to_frame()
        if data.empty:
            st.warning("Downloaded data is empty. Please check the ticker symbols and date range.")
            return pd.DataFrame()
        if data.index.tz is not None:
            data.index = data.index.tz_localize(None)
        data = data.fillna(method='ffill').fillna(method='bfill')
        if data.isnull().values.any():
            st.warning("Data contains missing values after filling. Some calculations may be affected.")
        return data
    except Exception as e:
        st.error(f"Error downloading data: {e}")
        return pd.DataFrame()

# ----------------------------
# Initialize Session State
# ----------------------------
if 'portfolios' not in st.session_state:
    st.session_state.portfolios = []
if 'backtest_results' not in st.session_state:
    st.session_state.backtest_results = {}
if 'step' not in st.session_state:
    st.session_state.step = "Configure Portfolio"
if 'edit_portfolio' not in st.session_state:
    st.session_state.edit_portfolio = None
if 'default_config' not in st.session_state:
    st.session_state.default_config = {
        'rf_rate': 0.02,
        'broker_fee': 0.0,
        'benchmark_symbol': '^GSPC'
    }

# ----------------------------
# Streamlit Layout
# ----------------------------
st.set_page_config(page_title="Portfolio Optimizer", layout="wide")

st.sidebar.header("Navigation")
step = st.sidebar.radio("Go to", [
    "Configure Portfolio",
    "Run Backtest",
    "Optimize Portfolio",
    "Monte Carlo Simulations",
    "Risk Analysis"
], index=["Configure Portfolio", "Run Backtest", "Optimize Portfolio", "Monte Carlo Simulations", "Risk Analysis"].index(st.session_state.get('step', "Configure Portfolio")), key="sidebar_radio")

tickers = get_tickers()

if step == "Configure Portfolio":
    st.title("🔧 Configure Portfolio")
    col1, col2 = st.columns([1,1])
    with col1:
        st.subheader("📁 Existing Portfolios")
        if st.session_state.portfolios:
            portfolio_names = [p['name'] for p in st.session_state.portfolios]
            selected_portfolio = st.selectbox("Select Portfolio to Edit/Delete/Duplicate", portfolio_names)
            if selected_portfolio:
                portfolio = next(p for p in st.session_state.portfolios if p['name'] == selected_portfolio)
                col_del, col_edit, col_dup = st.columns(3)
                with col_del:
                    if st.button("Delete Portfolio"):
                        st.session_state.portfolios = [p for p in st.session_state.portfolios if p['name'] != selected_portfolio]
                        st.success("Portfolio deleted.")
                        # Reset edit_portfolio if it was the deleted portfolio
                        if st.session_state.edit_portfolio == selected_portfolio:
                            st.session_state.edit_portfolio = None
                with col_edit:
                    if st.button("Edit Portfolio"):
                        st.session_state.edit_portfolio = selected_portfolio
                with col_dup:
                    with st.form("duplicate_portfolio_form"):
                        new_name = st.text_input("New Portfolio Name", f"{selected_portfolio}_Copy")
                        submitted_dup = st.form_submit_button("Confirm Duplicate")
                        if submitted_dup:
                            if not new_name:
                                st.error("Please provide a new portfolio name.")
                            elif new_name in [p['name'] for p in st.session_state.portfolios]:
                                st.error("Portfolio name already exists.")
                            else:
                                duplicated = portfolio.copy()
                                duplicated['name'] = new_name
                                st.session_state.portfolios.append(duplicated)
                                st.success("Portfolio duplicated successfully.")
        else:
            st.write("No portfolios saved.")
        # Edit Portfolio Section
        if st.session_state.edit_portfolio:
            st.markdown("---")
            st.subheader("✏️ Edit Portfolio")
            try:
                portfolio_to_edit = next(p for p in st.session_state.portfolios if p['name'] == st.session_state.edit_portfolio)
            except StopIteration:
                st.error("The portfolio you are trying to edit no longer exists.")
                st.session_state.edit_portfolio = None
            else:
                with st.form("edit_portfolio_form"):
                    name_edit = st.text_input("Portfolio Name", portfolio_to_edit['name'], help="Enter a unique name for your portfolio.")
                    start_date_edit = st.date_input(
                        "Start Date:",
                        portfolio_to_edit['start_date'],
                        help="Choose the start date for backtesting."
                    )
                    end_date_edit = st.date_input(
                        "End Date:",
                        portfolio_to_edit['end_date'],
                        help="Choose the end date for backtesting."
                    )
                    rf_rate_edit = st.number_input(
                        "Risk-Free Rate (%)",
                        0.0,
                        10.0,
                        portfolio_to_edit['rf_rate'] * 100,
                        help="Enter the risk-free rate as a percentage."
                    ) / 100
                    broker_fee_edit = st.number_input(
                        "Broker Fee (%)",
                        0.0,
                        10.0,
                        portfolio_to_edit['broker_fee'] * 100,
                        step=0.000001,
                        format="%.6f",
                        help="Enter the broker fee as a percentage per transaction."
                    ) / 100
                    benchmark_symbol_edit = st.text_input(
                        "Benchmark Symbol (e.g., ^GSPC)",
                        portfolio_to_edit['benchmark_symbol'],
                        help="Enter the ticker symbol for your benchmark index."
                    )
                    rebalance_freq_edit = st.selectbox(
                        "Rebalance Frequency",
                        ["Daily", "Weekly", "Monthly", "Quarterly", "Yearly"],
                        index=["Daily", "Weekly", "Monthly", "Quarterly", "Yearly"].index(
                            [k for k, v in frequency_mapping.items() if v == portfolio_to_edit['rebalance_freq']][0]
                        ),
                        help="Choose how often to rebalance the portfolio."
                    )
                    st.markdown("### 🛠️ Set Allocations")
                    # Set default selections as formatted strings based on existing tickers
                    selected_default = [format_asset_option(ticker, get_company_name(tickers, ticker)) for ticker in portfolio_to_edit['selected']]
                    ticker_df = get_tickers()
                    asset_options = ticker_df.apply(lambda row: format_asset_option(row['Ticker'], row['Company Name']), axis=1).tolist()
                    selected_edit = st.multiselect(
                        "Select Assets:",
                        options=asset_options,
                        default=selected_default,  # Use formatted strings for default
                        help="Choose the assets you want to include in your portfolio."
                    )
                    selected_tickers_edit = [option.split(' - ')[0] for option in selected_edit if ' - ' in option]
                    allocations_edit = []
                    if selected_tickers_edit:
                        for ticker in selected_tickers_edit:
                            alloc = st.number_input(
                                f"{ticker} Allocation (%)",
                                min_value=0.0,
                                max_value=100.0,
                                value=0.0,
                                step=0.000001,
                                format="%.6f",
                                key=f"alloc_edit_{ticker}"
                            )
                            allocations_edit.append(alloc)
                        current_total_edit = sum(allocations_edit)
                        st.markdown(f"**Total Allocation:** {current_total_edit:.6f}%")
                        if not np.isclose(current_total_edit, 100.0, atol=1e-4):
                            st.warning(f"Allocations must sum to 100%. Currently sum to {current_total_edit:.6f}%. Please adjust the allocations.")
                            allocations_edit = [alloc / current_total_edit * 100 for alloc in allocations_edit]
                    submitted_edit = st.form_submit_button("Save Changes")
                    if submitted_edit:
                        if not name_edit:
                            st.error("Please provide a portfolio name.")
                        elif name_edit != portfolio_to_edit['name'] and name_edit in [p['name'] for p in st.session_state.portfolios]:
                            st.error("Portfolio name already exists. Please choose a unique name.")
                        elif not selected_edit:
                            st.error("Please select at least one asset.")
                        total_allocation = sum(allocations_edit)
                        if not np.isclose(total_allocation, 100.0, atol=1e-4):
                            st.warning(f"Allocations must sum to 100%. Currently sum to {total_allocation:.6f}%. Adjusting allocations proportionally.")
                            allocations_edit = [alloc / total_allocation * 100 for alloc in allocations_edit]
                        portfolio_to_edit.update({
                            'name': name_edit,
                            'start_date': pd.to_datetime(start_date_edit),
                            'end_date': pd.to_datetime(end_date_edit),
                            'rf_rate': rf_rate_edit,
                            'broker_fee': broker_fee_edit,
                            'benchmark_symbol': benchmark_symbol_edit,
                            'rebalance_freq': frequency_mapping.get(rebalance_freq_edit, 'M'),
                            'selected': selected_tickers_edit,  # Corrected
                            'allocations': allocations_edit
                        })
                        st.success(f"Portfolio '{name_edit}' updated successfully!")
                        st.markdown("**Updated Allocations:**")
                        allocations_summary = {ticker: f"{alloc:.6f}%" for ticker, alloc in zip(selected_tickers_edit, allocations_edit)}
                        st.json(allocations_summary)
                        st.session_state.edit_portfolio = None
        st.markdown("---")
        st.subheader("💡 Example Portfolios")
        example_portfolios = [
            {
                'name': "Retirement Portfolio",
                'start_date': pd.to_datetime(datetime.today() - timedelta(days=365*10)),
                'end_date': pd.to_datetime(datetime.today()),
                'rf_rate': 0.02,
                'broker_fee': 0.0,
                'benchmark_symbol': '^GSPC',
                'rebalance_freq': 'M',
                'selected': ['AAPL', 'MSFT', 'GOOGL', 'JPM', 'XOM'],
                'allocations': [20.0, 20.0, 20.0, 20.0, 20.0]
            },
            {
                'name': "Tech-Heavy Portfolio",
                'start_date': pd.to_datetime(datetime.today() - timedelta(days=365*5)),
                'end_date': pd.to_datetime(datetime.today()),
                'rf_rate': 0.02,
                'broker_fee': 0.0,
                'benchmark_symbol': '^NASDAQ',
                'rebalance_freq': 'M',
                'selected': ['AAPL', 'MSFT', 'GOOGL', 'AMZN', 'FB'],
                'allocations': [25.0, 25.0, 20.0, 15.0, 15.0]
            },
            {
                'name': "Balanced Risk-Return Portfolio",
                'start_date': pd.to_datetime(datetime.today() - timedelta(days=365*7)),
                'end_date': pd.to_datetime(datetime.today()),
                'rf_rate': 0.02,
                'broker_fee': 0.0,
                'benchmark_symbol': '^GSPC',
                'rebalance_freq': 'M',
                'selected': ['AAPL', 'JNJ', 'V', 'PG', 'XOM'],
                'allocations': [20.0, 20.0, 20.0, 20.0, 20.0]
            },
            {
                'name': "Income-Focused Portfolio",
                'start_date': pd.to_datetime(datetime.today() - timedelta(days=365*7)),
                'end_date': pd.to_datetime(datetime.today()),
                'rf_rate': 0.02,
                'broker_fee': 0.0,
                'benchmark_symbol': '^DJI',
                'rebalance_freq': 'Q',
                'selected': ['PG', 'KO', 'JNJ', 'T', 'PFE'],
                'allocations': [20.0, 20.0, 20.0, 20.0, 20.0]
            },
            {
                'name': "Growth-Oriented Portfolio",
                'start_date': pd.to_datetime(datetime.today() - timedelta(days=365*3)),
                'end_date': pd.to_datetime(datetime.today()),
                'rf_rate': 0.02,
                'broker_fee': 0.0,
                'benchmark_symbol': '^IXIC',
                'rebalance_freq': 'M',
                'selected': ['TSLA', 'NVDA', 'AMD', 'FB', 'NFLX'],
                'allocations': [30.0, 25.0, 20.0, 15.0, 10.0]
            }
        ]
        
        for example in example_portfolios:
            with st.expander(example['name']):
                # Display portfolio details in a table
                df_example = pd.DataFrame({
                    'Attribute': ['Start Date', 'End Date', 'Risk-Free Rate (%)', 'Broker Fee (%)', 'Benchmark Symbol', 'Rebalance Frequency', 'Selected Assets', 'Allocations (%)'],
                    'Value': [
                        example['start_date'].strftime('%Y-%m-%d'),
                        example['end_date'].strftime('%Y-%m-%d'),
                        f"{example['rf_rate']*100:.2f}",
                        f"{example['broker_fee']*100:.6f}",
                        example['benchmark_symbol'],
                        example['rebalance_freq'],
                        ", ".join(example['selected']),
                        ", ".join([f"{alloc:.2f}" for alloc in example['allocations']])
                    ]
                })
                st.table(df_example)
                if st.button(f"Load {example['name']}", key=f"load_{example['name']}"):
                    if example['name'] in [p['name'] for p in st.session_state.portfolios]:
                        st.warning(f"Portfolio '{example['name']}' already exists.")
                    else:
                        st.session_state.portfolios.append(example.copy())
                        st.success(f"Portfolio '{example['name']}' loaded successfully!")
                        
    with col2:
        st.subheader("➕ Add New Portfolio")
        with st.form("add_portfolio_form"):

            name = st.text_input("Portfolio Name", "", help="Enter a unique name for your portfolio.")
            start_date = st.date_input(
                "Start Date:",
                datetime.today() - timedelta(days=365 * 5),
                help="Choose the start date for backtesting."
            )
            end_date = st.date_input(
                "End Date:",
                datetime.today(),
                help="Choose the end date for backtesting."
            )
            rf_rate = st.number_input(
                "Risk-Free Rate (%)",
                0.0,
                10.0,
                st.session_state.default_config['rf_rate'] * 100,
                help="Enter the risk-free rate as a percentage."
            ) / 100
            broker_fee = st.number_input(
                "Broker Fee (%)",
                0.0,
                10.0,
                st.session_state.default_config['broker_fee'] * 100,
                step=0.000001,
                format="%.6f",
                help="Enter the broker fee as a percentage per transaction."
            ) / 100
            benchmark_symbol = st.text_input(
                "Benchmark Symbol (e.g., ^GSPC)",
                st.session_state.default_config['benchmark_symbol'],
                help="Enter the ticker symbol for your benchmark index."
            )
            rebalance_freq = st.selectbox(
                "Rebalance Frequency",
                ["Daily", "Weekly", "Monthly", "Quarterly", "Yearly"],
                index=2,
                help="Choose how often to rebalance the portfolio."
            )
            st.markdown("### 🛠️ Set Allocations")
            ticker_df = get_tickers()
            asset_options = ticker_df.apply(lambda row: format_asset_option(row['Ticker'], row['Company Name']), axis=1).tolist()
            selected = st.multiselect(
                "Select Assets:",
                options=asset_options,
                help="Choose the assets you want to include in your portfolio."
            )
            # After selecting assets, extract tickers
            selected_tickers = [option.split(' - ')[0] for option in selected]
            allocations = []
            if selected_tickers:
                for ticker in selected_tickers:
                    alloc = st.number_input(
                        f"{ticker} Allocation (%)",
                        min_value=0.0,
                        max_value=100.0,
                        value=0.0,
                        step=0.000001,
                        format="%.6f",
                        key=f"allocations_new_{ticker}"
                    )
                    allocations.append(alloc)
                current_total = sum(allocations)
                st.markdown(f"**Total Allocation:** {current_total:.6f}%")
                if not np.isclose(current_total, 100.0, atol=1e-4):
                    st.warning(f"Allocations must sum to 100%. Currently sum to {current_total:.6f}%. Please adjust the allocations.")
            submitted = st.form_submit_button("Add Portfolio")
            if submitted:
                if not name:
                    st.error("Please provide a portfolio name.")
                elif name in [p['name'] for p in st.session_state.portfolios]:
                    st.error("Portfolio name already exists. Please choose a unique name.")
                elif not selected:
                    st.error("Please select at least one asset.")
                elif not np.isclose(sum(allocations), 100.0, atol=1e-4):
                    st.error(f"Allocations must sum to 100%. Currently sum to {sum(allocations):.6f}%.")
                else:
                        new_portfolio = {
                            'name': name,
                            'start_date': pd.to_datetime(start_date),
                            'end_date': pd.to_datetime(end_date),
                            'rf_rate': rf_rate,
                            'broker_fee': broker_fee,
                            'benchmark_symbol': benchmark_symbol,
                            'rebalance_freq': frequency_mapping.get(rebalance_freq, 'M'),
                            'selected': selected_tickers,
                            'allocations': allocations
                        }
                        if not np.isclose(sum(allocations), 100.0, atol=1e-4):
                            st.warning(f"Allocations must sum to 100%. Currently sum to {sum(allocations):.6f}%. Adjusting allocations proportionally.")
                            allocations_edit = [alloc / current_total_edit * 100 for alloc in allocations_edit]
                            new_portfolio['allocations'] = allocations
                        st.session_state.portfolios.append(new_portfolio)
                        st.success(f"Portfolio '{name}' added successfully!")
                st.markdown("**Allocations:**")
                allocations_summary = {ticker: f"{alloc}%" for ticker, alloc in zip(selected_tickers, allocations)}
                st.json(allocations_summary)

    # Debugging Information
    with st.expander("🔍 Debug Information"):
        st.write("Current Portfolios in Session State:")
        st.write(st.session_state.portfolios)

elif step == "Run Backtest":
    st.title("📊 Run Backtest")
    if st.session_state.portfolios:
        with st.form("compare_form"):
            st.subheader("🔍 Select Portfolios to Compare")
            comparison_type = st.selectbox(
                "Comparison Type",
                ["Portfolio vs Benchmark", "Portfolio vs Portfolio"]
            )
            if comparison_type == "Portfolio vs Benchmark":
                portfolio_names = [p['name'] for p in st.session_state.portfolios]
                portfolio1 = st.selectbox("Select Portfolio", portfolio_names)
                
                # Get benchmark suggestions based on selected tickers
                portfolio = next(p for p in st.session_state.portfolios if p['name'] == portfolio1)
                benchmarks = get_common_benchmarks(portfolio['selected'])
                benchmark_options = list(benchmarks.keys())
                benchmark_selected = st.selectbox(
                    "Select Benchmark",
                    options=benchmark_options,
                    help="Choose an appropriate benchmark based on your portfolio's asset composition."
                )
                if benchmark_selected == "CUSTOM":
                    benchmark_symbol = st.text_input("Enter Custom Benchmark Symbol:", "^GSPC")
                else:
                    benchmark_symbol = benchmarks.get(benchmark_selected, "^GSPC")  # Default to S&P 500 if not custom

                # Always update the benchmark_symbol in the portfolio
                portfolio['benchmark_symbol'] = benchmark_symbol

                st.write(f"**Benchmark:** {portfolio['benchmark_symbol']}")
                period = st.selectbox(
                    "Select Backtest Period",
                    ["Custom Range", "1 Month", "6 Months", "1 Year", "2 Years", "5 Years", "10 Years"],
                    help="Choose a predefined period for backtesting or set a custom date range."
                )

                today = datetime.today()

                if period == "Custom Range":
                    backtest_start_date = st.date_input(
                        "Backtest Start Date",
                        portfolio['start_date'],
                        help="Choose the start date for backtesting."
                    )
                    backtest_end_date = st.date_input(
                        "Backtest End Date",
                        portfolio['end_date'],
                        help="Choose the end date for backtesting."
                    )
                    if backtest_start_date > backtest_end_date:
                        st.error("Start date must be before end date.")
                        st.stop()
                else:
                    if period == "1 Month":
                        backtest_start_date = today - relativedelta(months=1)
                    elif period == "6 Months":
                        backtest_start_date = today - relativedelta(months=6)
                    elif period == "1 Year":
                        backtest_start_date = today - relativedelta(years=1)
                    elif period == "2 Years":
                        backtest_start_date = today - relativedelta(years=2)
                    elif period == "5 Years":
                        backtest_start_date = today - relativedelta(years=5)
                    elif period == "10 Years":
                        backtest_start_date = today - relativedelta(years=10)
                    backtest_end_date = today

                st.write(f"**Backtest Period:** {backtest_start_date.strftime('%Y-%m-%d')} to {backtest_end_date.strftime('%Y-%m-%d')}")
                    
            else:
                portfolio_names = [p['name'] for p in st.session_state.portfolios]
                portfolio1 = st.selectbox("Select First Portfolio", portfolio_names, key="p1")
                portfolio2 = st.selectbox("Select Second Portfolio", portfolio_names, key="p2")
            run = st.form_submit_button("Run Backtest")

        if run:
            with st.spinner("Running backtest..."):
                if comparison_type == "Portfolio vs Benchmark":
                    portfolio = next(p for p in st.session_state.portfolios if p['name'] == portfolio1)
                    
                    price_data = download_data(
                        portfolio['selected'],
                        backtest_start_date,
                        backtest_end_date
                    )
                    if price_data.empty:
                        st.error("No price data available for the selected portfolio.")
                    else:
                        available_selected = [ticker for ticker in portfolio['selected'] if ticker in price_data.columns]
                        missing_selected = list(set(portfolio['selected']) - set(available_selected))
                        if missing_selected:
                            st.warning(f"Excluded tickers with no data: {', '.join(missing_selected)}")
                        if not available_selected:
                            st.error(
                                "No selected tickers have available data for the chosen date range. "
                                "Please ensure that the ticker symbols are correct and data is available for the specified period."
                            )
                        else:
                            # Ensure allocations are correctly mapped to available tickers
                            allocations = [portfolio.get('allocations', [0.0]*len(portfolio.get('selected', [])))[i] 
                                          for i, ticker in enumerate(portfolio.get('selected', [])) if ticker in available_selected]
                            if not np.isclose(sum(allocations), 100.0, atol=1e-4):
                                st.warning(f"Allocations must sum to 100%. Currently sum to {sum(allocations):.6f}%. Adjusting allocations proportionally.")
                                allocations = [alloc / sum(allocations) * 100 for alloc in allocations]
                            portfolio['allocations'] = allocations
                            weights = np.array(allocations) / 100
                            returns, cum_returns = backtest(weights, price_data[available_selected],
                                                            portfolio['rebalance_freq'], portfolio['broker_fee'], debug=False)
                            if cum_returns.empty:
                                st.error("Cumulative returns are empty. Check the data and allocations.")
                            else:
                                benchmark_data = download_data([benchmark_symbol], backtest_start_date, backtest_end_date)
                            # Ensure benchmark_symbol is correctly set based on selection
                            if benchmark_symbol == "CUSTOM":
                                benchmark_symbol = st.text_input("Enter Custom Benchmark Symbol:", "^GSPC")
                            else:
                                benchmark_symbol = benchmarks.get(benchmark_selected, "^GSPC")  # Default to S&P 500 if not custom

                            if benchmark_symbol not in benchmark_data.columns:
                                st.error(f"The selected benchmark symbol '{benchmark_symbol}' does not have available data for the chosen period.")
                                st.stop()
                            else:
                                st.success(f"Benchmark '{benchmark_symbol}' data successfully downloaded and will be used for comparison.")
                                if not benchmark_data.empty:
                                    benchmark_returns = benchmark_data[benchmark_symbol].pct_change().dropna()
                                    benchmark_returns = benchmark_returns.reindex(returns.index, method='ffill').dropna()
                                    common_index = returns.index.intersection(benchmark_returns.index)
                                    if common_index.empty:
                                        st.error("No overlapping dates between portfolio returns and benchmark returns after alignment.")
                                        st.stop()
                                    returns = returns.loc[common_index]
                                    cum_returns = cum_returns.loc[common_index]
                                    benchmark_returns = benchmark_returns.loc[common_index]
                                    benchmark_cum_returns = (1 + benchmark_returns).cumprod()
                                    benchmark_metrics = calculate_simple_metrics(
                                        benchmark_returns, benchmark_cum_returns
                                    )
                                    extended_benchmark_metrics = calculate_portfolio_metrics(
                                        benchmark_returns, benchmark_cum_returns, portfolio['rf_rate'], None
                                    )
                                    benchmark_metrics.update(extended_benchmark_metrics)
                                else:
                                    benchmark_returns = None
                                    benchmark_cum_returns = pd.Series(dtype=float)
                                    benchmark_metrics = {
                                        'Start Balance': "N/A",
                                        'End Balance': "N/A",
                                        'Annualized Return (CAGR)': "N/A",
                                        'Best Year': "N/A",
                                        'Worst Year': "N/A",
                                        'Arithmetic Mean (Monthly)': "N/A",
                                        'Arithmetic Mean (Annualized)': "N/A",
                                        'Geometric Mean (Monthly)': "N/A",
                                        'Geometric Mean (Annualized)': "N/A",
                                        'Standard Deviation (Monthly)': "N/A",
                                        'Standard Deviation (Annualized)': "N/A",
                                        'Downside Deviation (Monthly)': "N/A",
                                        'Maximum Drawdown': "N/A",
                                        'Sharpe Ratio': "N/A",
                                        'Sortino Ratio': "N/A",
                                        'Gain/Loss Ratio': "N/A",
                                        'Skewness': "N/A",
                                        'Excess Kurtosis': "N/A",
                                        'Safe Withdrawal Rate': "N/A",
                                        'Perpetual Withdrawal Rate': "N/A",
                                        'Positive Periods': "N/A",
                                        'Benchmark Correlation': "N/A",
                                        'Beta': "N/A",
                                        'Alpha (annualized)': "N/A",
                                        'R2': "N/A",
                                        'Treynor Ratio': "N/A",
                                        'Calmar Ratio': "N/A",
                                        'Modigliani–Modigliani Measure': "N/A",
                                        'Information Ratio': "N/A",
                                        'Tracking Error': "N/A",
                                        'Active Return': "N/A",
                                        'Upside Capture Ratio': "N/A",
                                        'Downside Capture Ratio': "N/A"
                                    }

                        if not cum_returns.empty:
                            portfolio_metrics = calculate_portfolio_metrics(
                                returns, cum_returns, portfolio['rf_rate'], benchmark_returns
                            )

                            # Dashboard Header with Key Stats
                            with st.container():
                                st.markdown("### 🔑 Key Metrics")
                                key_metrics = {
                                    'Annualized Return (CAGR)': portfolio_metrics.get('Annualized Return (CAGR)', "N/A"),
                                    'Sharpe Ratio': portfolio_metrics.get('Sharpe Ratio', "N/A"),
                                    'Maximum Drawdown': portfolio_metrics.get('Maximum Drawdown', "N/A")
                                }
                                cols = st.columns(len(key_metrics))
                                for col, (metric, value) in zip(cols, key_metrics.items()):
                                    with col:
                                        st.metric(label=metric, value=value)

                            # Create Tabs for Organized Sections
                            tabs = st.tabs(["Overview", "Performance Statistics", "Advanced Metrics", "Drawdowns", "Visualizations"])

                            with tabs[0]:
                                st.write(f"**Portfolio Name:** {portfolio['name']}")
                                st.write(f"**Start Date:** {backtest_start_date.strftime('%Y-%m-%d')}")
                                st.write(f"**End Date:** {backtest_end_date.strftime('%Y-%m-%d')}")
                                st.write(f"**Benchmark:** {portfolio['benchmark_symbol']}")

                            with tabs[1]:
                                st.header("📊 Performance Statistics")
                                with st.container():
                                    st.subheader("🛠️ Basic Metrics")
                                    performance_data_stats = {
                                        'Metric': [
                                            'Start Balance',
                                            'End Balance',
                                            'Annualized Return (CAGR)',
                                            'Standard Deviation (Annualized)',
                                            'Best Year',
                                            'Worst Year',
                                            'Maximum Drawdown',
                                            'Sharpe Ratio',
                                            'Sortino Ratio',
                                            'Benchmark Correlation'
                                        ],
                                        'Portfolio': [
                                            portfolio_metrics['Start Balance'],
                                            portfolio_metrics['End Balance'],
                                            portfolio_metrics['Annualized Return (CAGR)'],
                                            portfolio_metrics['Standard Deviation (Annualized)'],
                                            portfolio_metrics['Best Year'],
                                            portfolio_metrics['Worst Year'],
                                            portfolio_metrics['Maximum Drawdown'],
                                            portfolio_metrics['Sharpe Ratio'],
                                            portfolio_metrics['Sortino Ratio'],
                                            portfolio_metrics['Benchmark Correlation']
                                        ],
                                        'Benchmark': [
                                            benchmark_metrics['Start Balance'],
                                            benchmark_metrics['End Balance'],
                                            benchmark_metrics['Annualized Return (CAGR)'],
                                            benchmark_metrics['Standard Deviation (Annualized)'],
                                            benchmark_metrics['Best Year'],
                                            benchmark_metrics['Worst Year'],
                                            benchmark_metrics['Maximum Drawdown'],
                                            benchmark_metrics['Sharpe Ratio'],
                                            benchmark_metrics['Sortino Ratio'],
                                            benchmark_metrics['Benchmark Correlation']
                                        ]
                                    }
                                    performance_df_stats = pd.DataFrame(performance_data_stats).set_index('Metric')
                                    st.table(performance_df_stats)

                            with tabs[2]:
                                st.header("📉 Advanced Metrics")
                                with st.container():
                                    st.subheader("📈 Advanced Metrics")
                                    performance_data_advanced = {
                                        'Metric': list(portfolio_metrics.keys()),
                                        'Portfolio': list(portfolio_metrics.values()),
                                        'Benchmark': list(benchmark_metrics.values())
                                    }
                                    performance_df_advanced = pd.DataFrame(performance_data_advanced).set_index('Metric')
                                    st.table(performance_df_advanced)

                            with tabs[3]:
                                st.header("📉 Detailed Drawdowns")
                                st.markdown("### 📈 Drawdowns for Portfolio")
                                portfolio_drawdowns = get_drawdown_details(cum_returns)
                                if portfolio_drawdowns:
                                    portfolio_drawdowns_df = pd.DataFrame(portfolio_drawdowns)
                                    st.table(portfolio_drawdowns_df)
                                else:
                                    st.write("No drawdowns detected for the portfolio.")

                                if benchmark_returns is not None and not benchmark_returns.empty:
                                    st.markdown("### 📈 Drawdowns for Benchmark")
                                    benchmark_drawdowns = get_drawdown_details(benchmark_cum_returns)
                                    if benchmark_drawdowns:
                                        benchmark_drawdowns_df = pd.DataFrame(benchmark_drawdowns)
                                        st.table(benchmark_drawdowns_df)
                                    else:
                                        st.write("No drawdowns detected for the benchmark.")
                                else:
                                    st.write("Benchmark data not available for drawdown analysis.")

                            with tabs[4]:
                                st.header("📊 Visualizations")
                                
                                # Create Subsections Using Separate Columns or Expanders for Better Organization
                                st.subheader("📈 Growth Comparison")
                                plot_growth_comparison(cum_returns, benchmark_cum_returns)
                                
                                st.subheader("📉 Drawdown Comparison")
                                portfolio_drawdown_series = (cum_returns / cum_returns.expanding().max() - 1) * 100
                                benchmark_drawdown_series = (benchmark_cum_returns / benchmark_cum_returns.expanding().max() - 1) * 100
                                plot_drawdown_comparison(portfolio_drawdown_series, benchmark_drawdown_series)
                                
                                # Organize Rolling Return and Box Plot Side by Side
                                col1, col2 = st.columns(2)
                                with col1:
                                    st.subheader("⏱️ Rolling Annualized Return Comparison")
                                    plot_rolling_return_comparison(cum_returns, benchmark_cum_returns, window=252)
                                with col2:
                                    st.subheader("📦 Returns Distribution Box Plot")
                                    if isinstance(returns, pd.DataFrame):
                                        melted_returns = returns.reset_index().melt(id_vars='Date', var_name='Asset', value_name='Return')
                                        fig_box_plot = px.box(
                                            melted_returns,
                                            x='Asset',
                                            y='Return',
                                            title='Returns Distribution Box Plot',
                                            labels={'Return': 'Returns', 'Asset': 'Asset'},
                                            points='all',
                                            hover_data=['Return'],
                                            template='plotly_dark',
                                            color='Asset',
                                            color_discrete_sequence=px.colors.qualitative.Pastel
                                        )
                                        st.plotly_chart(fig_box_plot, use_container_width=True)
                                        st.markdown("""
                                        **Interpretation:** The box plot visualizes the distribution of returns for each asset in the portfolio.  
                                        - **Boxes:** Represent the interquartile range (IQR).  
                                        - **Median Line:** Indicates the median return.  
                                        - **Whiskers:** Show the range of the data.  
                                        - **Outliers:** Displayed as individual points.
                                        """)
                                    elif isinstance(returns, pd.Series):
                                        fig_box_plot = px.box(
                                            returns.to_frame(name='Return'),
                                            y='Return',
                                            title='Returns Distribution Box Plot',
                                            labels={'Return': 'Returns'},
                                            points='all',
                                            hover_data=['Return'],
                                            template='plotly_dark',
                                            color_discrete_sequence=px.colors.qualitative.Pastel
                                        )
                                        st.plotly_chart(fig_box_plot, use_container_width=True)
                                        st.markdown("""
                                        **Interpretation:** The box plot visualizes the distribution of returns for the portfolio.  
                                        - **Boxes:** Represent the interquartile range (IQR).  
                                        - **Median Line:** Indicates the median return.  
                                        - **Whiskers:** Show the range of the data.  
                                        - **Outliers:** Displayed as individual points.
                                        """)
                                    else:
                                        st.warning("Returns data is neither a DataFrame nor a Series.")
                                
                                # Organize Heatmaps Side by Side
                                col3, col4 = st.columns(2)
                                with col3:
                                    st.subheader("🔥 Correlation Heatmap of Assets")
                                    plot_correlation_heatmap(price_data[available_selected].pct_change().dropna())
                                
                                with col4:
                                    st.subheader("🔥 Cumulative Returns Heatmap")
                                    try:
                                        cum_returns_normalized = cum_returns / cum_returns.max()
                                        fig_cum_heatmap = px.imshow(
                                            cum_returns_normalized.to_frame().T,
                                            labels=dict(x="Date", y="Portfolio", color="Normalized Cumulative Return"),
                                            title="Cumulative Returns Heatmap",
                                            aspect="auto",
                                            color_continuous_scale='Viridis',
                                            template='plotly_white'
                                        )
                                        st.plotly_chart(fig_cum_heatmap, use_container_width=True)
                                        st.markdown("""
                                        **Interpretation:** The heatmap visualizes the normalized cumulative returns over time,  
                                        allowing for an intuitive comparison of portfolio performance across different periods.
                                        """)
                                    except Exception as e:
                                        st.error(f"Error plotting cumulative returns heatmap: {e}")
                                
                                # Portfolio Allocation Pie Chart Full Width
                                st.subheader("🥧 Portfolio Allocation Pie Chart")
                                plot_allocation_pie(weights, available_selected)

                                # Final Score
                                st.markdown("### 📝 Final Score")
                                portfolio_score = calculate_final_score(portfolio_metrics, benchmark_metrics)
                                benchmark_score = calculate_final_score(benchmark_metrics, portfolio_metrics)
                                st.write(f"**Portfolio:** {portfolio_score:.2f} / 100")
                                st.write(f"**Benchmark:** {benchmark_score:.2f} / 100")

                                # Store results
                                st.session_state.backtest_results = {
                                    'returns': returns,
                                    'cum_returns': cum_returns,
                                    'weights_a': weights,  # Renamed from 'weights' to 'weights_a'
                                    'price_data': price_data[available_selected],
                                    'metrics': portfolio_metrics,
                                    'benchmark_metrics': benchmark_metrics,
                                    'performance_df_stats': performance_df_stats,
                                    'performance_df_advanced': performance_df_advanced,
                                    'benchmark_cum_returns': benchmark_cum_returns
                                }
                                st.success("Backtest completed!")
                elif comparison_type == "Portfolio vs Portfolio":
                    portfolio_names = [p['name'] for p in st.session_state.portfolios]
                    if portfolio1 == portfolio2:
                        st.error("Please select two different portfolios for comparison.")
                    else:
                        portfolio_a = next(p for p in st.session_state.portfolios if p['name'] == portfolio1)
                        portfolio_b = next(p for p in st.session_state.portfolios if p['name'] == portfolio2)

                        # Backtest Portfolio A
                        price_data_a = download_data(
                            portfolio_a['selected'],
                            portfolio_a['start_date'],
                            portfolio_a['end_date']
                        )
                        if price_data_a.empty:
                            st.error(f"No price data available for portfolio '{portfolio_a['name']}'.")
                            st.stop()
                        available_a = [ticker for ticker in portfolio_a['selected'] if ticker in price_data_a.columns]
                        missing_a = list(set(portfolio_a['selected']) - set(available_a))
                        if missing_a:
                            st.warning(f"Excluded tickers from '{portfolio_a['name']}': {', '.join(missing_a)}")
                        if not available_a:
                            st.error(f"No valid tickers for portfolio '{portfolio_a['name']}'.")
                            st.stop()
                        allocs_a = [portfolio_a['allocations'][i] for i, ticker in enumerate(portfolio_a['selected']) if
                                   ticker in available_a]
                        if not np.isclose(sum(allocs_a), 100.0, atol=1e-6):
                            allocs_a = [a / sum(allocs_a) * 100 for a in allocs_a]
                            st.warning(f"Allocations for '{portfolio_a['name']}' adjusted to sum to 100%.")
                        weights_a = np.array(allocs_a) / 100
                        returns_a, cum_returns_a = backtest(weights_a, price_data_a[available_a],
                                                          portfolio_a['rebalance_freq'], portfolio_a['broker_fee'], debug=False)

                        # Backtest Portfolio B
                        price_data_b = download_data(
                            portfolio_b['selected'],
                            portfolio_b['start_date'],
                            portfolio_b['end_date']
                        )
                        if price_data_b.empty:
                            st.error(f"No price data available for portfolio '{portfolio_b['name']}'.")
                            st.stop()
                        available_b = [ticker for ticker in portfolio_b['selected'] if ticker in price_data_b.columns]
                        missing_b = list(set(portfolio_b['selected']) - set(available_b))
                        if missing_b:
                            st.warning(f"Excluded tickers from '{portfolio_b['name']}': {', '.join(missing_b)}")
                        if not available_b:
                            st.error(f"No valid tickers for portfolio '{portfolio_b['name']}'.")
                            st.stop()
                        allocs_b = [portfolio_b['allocations'][i] for i, ticker in enumerate(portfolio_b['selected']) if
                                   ticker in available_b]
                        if not np.isclose(sum(allocs_b), 100.0, atol=1e-6):
                            allocs_b = [a / sum(allocs_b) * 100 for a in allocs_b]
                            st.warning(f"Allocations for '{portfolio_b['name']}' adjusted to sum to 100%.")
                        weights_b = np.array(allocs_b) / 100
                        returns_b, cum_returns_b = backtest(weights_b, price_data_b[available_b],
                                                          portfolio_b['rebalance_freq'], portfolio_b['broker_fee'], debug=False)

                        if cum_returns_a.empty or cum_returns_b.empty:
                            st.error("One of the portfolios has empty cumulative returns. Check the data and allocations.")
                            st.stop()

                        # Align dates
                        common_index = cum_returns_a.index.intersection(cum_returns_b.index)
                        if common_index.empty:
                            st.error("No overlapping dates between the two portfolios after alignment.")
                            st.stop()
                        returns_a = returns_a.loc[common_index]
                        cum_returns_a = cum_returns_a.loc[common_index]
                        returns_b = returns_b.loc[common_index]
                        cum_returns_b = cum_returns_b.loc[common_index]

                        if cum_returns_a.empty or cum_returns_b.empty:
                            st.error("No overlapping dates between the two portfolios after alignment.")
                            st.stop()

                        # Calculate metrics
                        metrics_a = calculate_portfolio_metrics(
                            returns_a, cum_returns_a, portfolio_a['rf_rate'], returns_b.mean(axis=1)
                        )
                        metrics_b = calculate_portfolio_metrics(
                            returns_b, cum_returns_b, portfolio_b['rf_rate'], returns_a.mean(axis=1)
                        )

                        # Dashboard Header with Key Stats
                        with st.container():
                            st.markdown("### 🔑 Key Metrics")
                            key_metrics_a = {
                                'Annualized Return (CAGR)': metrics_a.get('Annualized Return (CAGR)', "N/A"),
                                'Sharpe Ratio': metrics_a.get('Sharpe Ratio', "N/A"),
                                'Maximum Drawdown': metrics_a.get('Maximum Drawdown', "N/A")
                            }
                            key_metrics_b = {
                                'Annualized Return (CAGR)': metrics_b.get('Annualized Return (CAGR)', "N/A"),
                                'Sharpe Ratio': metrics_b.get('Sharpe Ratio', "N/A"),
                                'Maximum Drawdown': metrics_b.get('Maximum Drawdown', "N/A")
                            }
                            cols = st.columns(len(key_metrics_a))
                            for col, (metric, value) in zip(cols, key_metrics_a.items()):
                                with col:
                                    st.metric(label=f"{portfolio_a['name']} {metric}", value=value)
                            for col, (metric, value) in zip(cols, key_metrics_b.items()):
                                with col:
                                    st.metric(label=f"{portfolio_b['name']} {metric}", value=value)

                        # Create Tabs for Organized Sections
                        tabs = st.tabs(["Overview", "Performance Statistics", "Advanced Metrics", "Drawdowns", "Visualizations"])

                        with tabs[0]:
                            st.header("📈 Portfolio Performance Overview")
                            st.write(f"**Portfolio Name:** {portfolio['name']}")
                            st.write(f"**Start Date:** {portfolio['start_date'].strftime('%Y-%m-%d')}")
                            st.write(f"**End Date:** {portfolio['end_date'].strftime('%Y-%m-%d')}")
                            st.write(f"**Benchmark:** {portfolio['benchmark_symbol']}")

                        with tabs[1]:
                            st.header("📊 Performance Statistics")
                            with st.expander("🛠️ Basic Metrics", expanded=True):
                                performance_data_stats = {
                                    'Metric': [
                                        'Start Balance',
                                        'End Balance',
                                        'Annualized Return (CAGR)',
                                        'Standard Deviation (Annualized)',
                                        'Best Year',
                                        'Worst Year',
                                        'Maximum Drawdown',
                                        'Sharpe Ratio',
                                        'Sortino Ratio',
                                        'Benchmark Correlation'
                                    ],
                                    portfolio_a['name']: [
                                        metrics_a['Start Balance'],
                                        metrics_a['End Balance'],
                                        metrics_a['Annualized Return (CAGR)'],
                                        metrics_a['Standard Deviation (Annualized)'],
                                        metrics_a['Best Year'],
                                        metrics_a['Worst Year'],
                                        metrics_a['Maximum Drawdown'],
                                        metrics_a['Sharpe Ratio'],
                                        metrics_a['Sortino Ratio'],
                                        metrics_a['Benchmark Correlation']
                                    ],
                                    portfolio_b['name']: [
                                        metrics_b['Start Balance'],
                                        metrics_b['End Balance'],
                                        metrics_b['Annualized Return (CAGR)'],
                                        metrics_b['Standard Deviation (Annualized)'],
                                        metrics_b['Best Year'],
                                        metrics_b['Worst Year'],
                                        metrics_b['Maximum Drawdown'],
                                        metrics_b['Sharpe Ratio'],
                                        metrics_b['Sortino Ratio'],
                                        metrics_b['Benchmark Correlation']
                                    ]
                                }
                                performance_df_stats = pd.DataFrame(performance_data_stats).set_index('Metric')
                                st.table(performance_df_stats)

                        with tabs[2]:
                            st.header("📉 Advanced Metrics")
                            with st.expander("📈 Advanced Metrics", expanded=False):
                                performance_data_advanced = {
                                    'Metric': list(metrics_a.keys()),
                                    portfolio_a['name']: list(metrics_a.values()),
                                    portfolio_b['name']: list(metrics_b.values())
                                }
                                performance_df_advanced = pd.DataFrame(performance_data_advanced).set_index('Metric')
                                st.table(performance_df_advanced)

                        with tabs[3]:
                            st.header("📉 Drawdowns")
                            st.markdown("### 📉 Detailed Drawdowns")
                            st.markdown(f"#### 📉 Drawdowns for {portfolio_a['name']}")
                            drawdowns_a = get_drawdown_details(cum_returns_a)
                            if drawdowns_a:
                                drawdowns_a_df = pd.DataFrame(drawdowns_a)
                                st.table(drawdowns_a_df)
                            else:
                                st.write("No drawdowns detected.")

                            st.markdown(f"#### 📉 Drawdowns for {portfolio_b['name']}")
                            drawdowns_b = get_drawdown_details(cum_returns_b)
                            if drawdowns_b:
                                drawdowns_b_df = pd.DataFrame(drawdowns_b)
                                st.table(drawdowns_b_df)
                            else:
                                st.write("No drawdowns detected.")

                        with tabs[4]:
                            st.header("📊 Visualizations")
                            # Growth Comparison
                            fig_growth = px.line(title='Growth Comparison')
                            fig_growth.add_scatter(x=cum_returns_a.index, y=cum_returns_a, mode='lines', name=portfolio_a['name'])
                            fig_growth.add_scatter(x=cum_returns_b.index, y=cum_returns_b, mode='lines', name=portfolio_b['name'])
                            st.plotly_chart(fig_growth, use_container_width=True)

                            # Drawdown Comparison
                            portfolio_drawdown_series_a = (cum_returns_a / cum_returns_a.expanding().max() - 1) * 100
                            portfolio_drawdown_series_b = (cum_returns_b / cum_returns_b.expanding().max() - 1) * 100
                            fig_drawdown = px.line(title='Drawdown Comparison')
                            fig_drawdown.add_scatter(x=portfolio_drawdown_series_a.index, y=portfolio_drawdown_series_a, mode='lines', name=portfolio_a['name'])
                            fig_drawdown.add_scatter(x=portfolio_drawdown_series_b.index, y=portfolio_drawdown_series_b, mode='lines', name=portfolio_b['name'])
                            st.plotly_chart(fig_drawdown, use_container_width=True)

                            # **New Additional Visualizations**
                            # Box Plot for Returns Distribution
                            st.subheader("📦 Returns Distribution Box Plot")
                            fig_box_plot_a = px.box(returns_a, title=f'Returns Distribution Box Plot for {portfolio_a["name"]}', labels={'value': 'Returns', 'variable': 'Asset'})
                            fig_box_plot_b = px.box(returns_b, title=f'Returns Distribution Box Plot for {portfolio_b["name"]}', labels={'value': 'Returns', 'variable': 'Asset'})
                            st.plotly_chart(fig_box_plot_a, use_container_width=True)
                            st.plotly_chart(fig_box_plot_b, use_container_width=True)

                            # Heatmap of Correlations Between Assets
                            st.subheader("🔥 Correlation Heatmap of Assets")
                            plot_correlation_heatmap(price_data_a[available_a].pct_change().dropna())
                            plot_correlation_heatmap(price_data_b[available_b].pct_change().dropna())

                            # Portfolio Allocation Pie Chart
                            st.subheader("🥧 Portfolio Allocation Pie Chart")
                            plot_allocation_pie(weights_a, available_a, title=f"{portfolio_a['name']} Allocation")
                            plot_allocation_pie(weights_b, available_b, title=f"{portfolio_b['name']} Allocation")

                            # Final Score
                            st.markdown("### 📝 Final Score")
                            score_a = calculate_final_score(metrics_a, metrics_b)
                            score_b = calculate_final_score(metrics_b, metrics_a)
                            st.write(f"**{portfolio_a['name']}:** {score_a:.2f} / 100")
                            st.write(f"**{portfolio_b['name']}:** {score_b:.2f} / 100")

                            # Store results
                            st.session_state.backtest_results = {
                                'returns_a': returns_a,
                                'cum_returns_a': cum_returns_a,
                                'weights_a': weights_a,
                                'price_data_a': price_data_a[available_a],
                                'metrics_a': metrics_a,
                                'returns_b': returns_b,
                                'cum_returns_b': cum_returns_b,
                                'weights_b': weights_b,
                                'price_data_b': price_data_b[available_b],
                                'metrics_b': metrics_b,
                                'performance_df_stats': performance_df_stats,
                                'performance_df_advanced': performance_df_advanced,
                                'benchmark_cum_returns': pd.Series(dtype=float)
                            }
                            st.success("Backtest completed!")
    else:
        st.warning("Please add at least one portfolio to run backtest.")

    # ----------------------------
    # Onboarding Process
    # ----------------------------
    if 'first_time_user' not in st.session_state:
        st.session_state.first_time_user = True

    if st.session_state.first_time_user:
        st.info("Welcome! Let's get you started with your first portfolio.")
        st.write("1. Configure your portfolio by entering a name, start date, end date, and other details.")
        st.write("2. Select the assets you want to include and set their allocations.")
        st.write("3. Run a backtest to see how your portfolio performs.")
        st.write("4. Compare your portfolio with a benchmark or another portfolio.")
        st.write("5. Explore the performance metrics and visualizations.")
        st.session_state.first_time_user = False

    # ----------------------------
    # Documentation
    # ----------------------------
    with st.expander("📖 Documentation"):
        st.markdown("""
        ### Documentation

        **Portfolio Configuration:**
        - **Portfolio Name:** Enter a unique name for your portfolio.
        - **Start Date:** Choose the start date for backtesting.
        - **End Date:** Choose the end date for backtesting.
        - **Risk-Free Rate:** Enter the risk-free rate as a percentage.
        - **Broker Fee:** Enter the broker fee as a percentage per transaction.
        - **Benchmark Symbol:** Enter the ticker symbol for your benchmark index (e.g., ^GSPC for S&P 500).
        - **Rebalance Frequency:** Choose how often to rebalance the portfolio.
        - **Select Assets:** Choose the assets you want to include in your portfolio.
        - **Set Allocations:** Set the allocation percentages for each selected asset.

        **Running Backtests:**
        - Select the portfolios you want to compare.
        - Choose the comparison type (Portfolio vs Benchmark or Portfolio vs Portfolio).
        - Click "Run Backtest" to see the results.

        **Performance Metrics:**
        - **Start Balance:** Initial balance at the start of the backtest.
        - **End Balance:** Final balance at the end of the backtest.
        - **Annualized Return (CAGR):** Compound Annual Growth Rate.
        - **Standard Deviation (Annualized):** Annualized standard deviation of returns.
        - **Best Year:** Best annual return.
        - **Worst Year:** Worst annual return.
        - **Maximum Drawdown:** Largest peak-to-trough decline.
        - **Sharpe Ratio:** Risk-adjusted return.
        - **Sortino Ratio:** Risk-adjusted return considering only downside volatility.
        - **Benchmark Correlation:** Correlation with the benchmark.

        **Visualizations:**
        - **Growth Comparison:** Cumulative returns of the portfolio and benchmark.
        - **Drawdown Comparison:** Drawdowns of the portfolio and benchmark.
        - **Monthly Returns Distribution:** Distribution of monthly returns.
        - **Cumulative Returns Heatmap:** Heatmap of cumulative returns over time.

        **Final Score:**
        - Compares the portfolio's performance metrics against the benchmark or another portfolio.
        """)
        
elif step == "Optimize Portfolio":
    st.title("🔧 Optimize Portfolio")
    if st.session_state.portfolios:
        portfolio_names = [p['name'] for p in st.session_state.portfolios]
        selected_portfolio = st.selectbox("Select Portfolio to Optimize", portfolio_names)

        portfolio = next(p for p in st.session_state.portfolios if p['name'] == selected_portfolio)
        price_data = download_data(
            portfolio['selected'],
            portfolio['start_date'],
            portfolio['end_date']
        )
        if price_data.empty:
            st.error("No price data available for the selected portfolio.")
        else:
            available_selected = [ticker for ticker in portfolio['selected'] if ticker in price_data.columns]
            missing_selected = list(set(portfolio['selected']) - set(available_selected))
            if missing_selected:
                st.warning(f"Excluded tickers with no data: {', '.join(missing_selected)}")
            if not available_selected:
                st.error(
                    "No selected tickers have available data for the chosen date range. "
                    "Please ensure that the ticker symbols are correct and data is available for the specified period."
                )
            else:
                returns = price_data[available_selected].pct_change().dropna()
                with st.form("optimization_form"):
                    st.markdown("### 🛠️ Set Optimization Constraints")
                    # Interactive Sliders
                    max_weight = st.slider(
                        "Maximum Weight per Asset (%)",
                        min_value=0.0,
                        max_value=100.0,
                        value=100.0,
                        step=0.1,
                        help="Set the maximum allocation percentage for any single asset."
                    ) / 100  # Convert to decimal

                    min_weight = st.slider(
                        "Minimum Weight per Asset (%)",
                        min_value=0.0,
                        max_value=100.0,
                        value=0.0,
                        step=0.1,
                        help="Set the minimum allocation percentage for any single asset."
                    ) / 100  # Convert to decimal

                    # **Added Validation Check**
                    if min_weight > max_weight:
                        st.error("Minimum weight cannot exceed maximum weight.")
                        st.stop()  # Prevent further execution

                    # **Added: Multi-Objective Selection**
                    objective_options = ["Sharpe Ratio", "Minimum Variance", "Maximum Return", "Minimum Drawdown"]
                    selected_objectives = st.multiselect(
                        "Select Optimization Objectives",
                        options=objective_options,
                        default=["Sharpe Ratio"],
                        help="Choose one or more objectives to optimize your portfolio."
                    )

                    target_return = st.number_input(
                        "Target Annual Return (%)",
                        min_value=0.0,
                        max_value=100.0,
                        value=10.0,
                        step=0.1,
                        help="Set the target annual return for portfolio optimization."
                    ) / 100  # Convert to decimal

                    submitted_optimize = st.form_submit_button("Run Optimization")

                    if submitted_optimize:
                        if not selected_objectives:
                            st.error("Please select at least one optimization objective.")
                        else:
                            objective_map = {
                                'Sharpe Ratio': 'sharpe',
                                'Minimum Variance': 'min_variance',
                                'Maximum Return': 'max_return',
                                'Minimum Drawdown': 'min_drawdown'
                            }
                            objectives_selected = [objective_map[obj] for obj in selected_objectives]
                            with st.spinner("Optimizing portfolio..."):
                                optimized_weights = optimize_portfolio(
                                    returns,
                                    objectives=objectives_selected,
                                    rf=portfolio['rf_rate'],
                                    max_weight=max_weight,
                                    min_weight=min_weight,
                                    target_return=target_return
                                )
                                if optimized_weights is not None:
                                    # **Added Allocation Sum Check**
                                    total_allocation = np.sum(optimized_weights)
                                    if not np.isclose(total_allocation, 1.0, atol=1e-4):
                                        st.warning(f"Optimized allocations sum to {total_allocation*100:.2f}%. Adjusting proportionally.")
                                        optimized_weights = optimized_weights / total_allocation

                                    st.success("Optimization completed successfully!")

                                    # Display Optimized Weights
                                    st.subheader("📈 Optimized Weights")
                                    weights_df = pd.DataFrame(optimized_weights, index=available_selected, columns=["Weight"])
                                    weights_df['Weight'] = weights_df['Weight'].apply(lambda x: f"{x:.2%}")
                                    st.table(weights_df)

                                    # Plot Allocation Pie Chart
                                    plot_allocation_pie(optimized_weights, available_selected, title="Optimized Portfolio Allocation")

                                    # Display Performance Metrics
                                    st.subheader("📈 Optimized Portfolio Performance Metrics")
                                    optimized_return = np.dot(optimized_weights, returns.mean()) * 252
                                    optimized_volatility = np.sqrt(np.dot(optimized_weights.T, np.dot(returns.cov() * 252, optimized_weights)))
                                    optimized_sharpe = (optimized_return - portfolio['rf_rate']) / optimized_volatility if optimized_volatility > 1e-6 else np.nan

                                    # After calculating optimized_returns and cum_returns (if not already present)
                                    cum_returns_optimized = (1 + returns.dot(optimized_weights)).cumprod()

                                    # Calculate Maximum Drawdown
                                    max_drawdown = drawdown(cum_returns_optimized)

                                    # Include Maximum Drawdown in optimized metrics
                                    optimized_metrics = {
                                        'Expected Annual Return (%)': f"{optimized_return * 100:.2f}%",
                                        'Annualized Volatility (%)': f"{optimized_volatility * 100:.2f}%",
                                        'Sharpe Ratio': f"{optimized_sharpe:.2f}",
                                        'Maximum Drawdown (%)': f"{max_drawdown * 100:.2f}%"
                                    }

                                    optimized_metrics_df = pd.DataFrame(list(optimized_metrics.items()), columns=['Metric', 'Value'])
                                    st.table(optimized_metrics_df)

                                    # Store optimized weights and metrics in session state
                                    st.session_state.backtest_results.update({
                                        'optimized_weights': optimized_weights,
                                        'optimized_return': optimized_return,
                                        'optimized_volatility': optimized_volatility,
                                        'optimized_sharpe': optimized_sharpe
                                    })
                                else:
                                    st.error("Optimization did not return any weights.")
    else:
        st.warning("Please add at least one portfolio to optimize.")

elif step == "Monte Carlo Simulations":
    st.title("📈 Monte Carlo Simulations")
    if st.session_state.portfolios:
        portfolio_names = [p['name'] for p in st.session_state.portfolios]
        selected_portfolio = st.selectbox("Select Portfolio for Monte Carlo Simulations", portfolio_names)
        if selected_portfolio:
            portfolio = next(p for p in st.session_state.portfolios if p['name'] == selected_portfolio)
            price_data = download_data(
                portfolio['selected'],
                portfolio['start_date'],
                portfolio['end_date']
            )
            if price_data.empty:
                st.error("No price data available for the selected portfolio.")
            else:
                available_selected = [ticker for ticker in portfolio['selected'] if ticker in price_data.columns]
                missing_selected = list(set(portfolio['selected']) - set(available_selected))
                if missing_selected:
                    st.warning(f"Excluded tickers with no data: {', '.join(missing_selected)}")
                if not available_selected:
                    st.error("No selected tickers have available data for the chosen date range. "
                             "Please ensure that the ticker symbols are correct and data is available for the specified period.")
                else:
                    returns = price_data[available_selected].pct_change().dropna()
                    
                    # Add Simulation Scenario Selection
                    st.subheader("🔍 Select Simulation Scenario")
                    scenario = st.selectbox(
                        "Choose a Simulation Scenario",
                        ["Average Market Performance", "Best Case", "Worst Case"],
                        help="Select a predefined simulation scenario to adjust the simulation parameters accordingly."
                    )

                    # Adjust Simulation Parameters Based on Scenario
                    if scenario == "Best Case":
                        adjusted_mean = returns.mean() * 1.2  # Increase mean returns by 20%
                        adjusted_cov = returns.cov() * 0.8    # Decrease volatility by 20%
                    elif scenario == "Worst Case":
                        adjusted_mean = returns.mean() * 0.8  # Decrease mean returns by 20%
                        adjusted_cov = returns.cov() * 1.2    # Increase volatility by 20%
                    else:
                        adjusted_mean = returns.mean()        # Average scenario
                        adjusted_cov = returns.cov()

                    num_simulations = st.number_input("Number of Simulations", min_value=1, max_value=10000, value=1000)
                    if st.button("Run Simulations"):
                        with st.spinner("Running Monte Carlo simulations..."):
                            # Use adjusted_mean and adjusted_cov based on scenario
                            simulated_returns = monte_carlo_simulation(returns, num_simulations, periods=252, mean_returns=adjusted_mean, cov_matrix=adjusted_cov)
                            simulated_returns = np.array(simulated_returns)  # Convert to NumPy array
                            
                            # Describe Simulated Returns
                            st.write("Simulated Portfolio Returns:")
                            st.write(pd.Series(simulated_returns).describe())
                            
                            # Combined Simulated Returns Distribution with Cumulative Overlay
                            fig_simulated_returns = px.histogram(
                                simulated_returns, 
                                nbins=50, 
                                title='Simulated Portfolio Returns Distribution',
                                labels={'value': 'Simulated Returns', 'count': 'Frequency'},
                                opacity=0.6,
                                name='Simulated Returns'
                            )

                            # Add Cumulative Distribution Line
                            sorted_returns = np.sort(simulated_returns)
                            cdf = np.linspace(0, 1, len(sorted_returns))
                            fig_simulated_returns.add_scatter(x=sorted_returns, y=cdf * simulated_returns.size, mode='lines', name='CDF', yaxis='y2')

                            # Update Layout to Include Secondary Y-Axis
                            fig_simulated_returns.update_layout(
                                yaxis2=dict(
                                    overlaying='y',
                                    side='right',
                                    title='Cumulative Frequency'
                                ),
                                legend=dict(x=0.7, y=0.95)
                            )

                            # Identify Most Optimistic and Pessimistic Outcomes
                            best_return = simulated_returns.max()
                            worst_return = simulated_returns.min()
                            fig_simulated_returns.add_vline(x=best_return, line_dash="dash", line_color="green", annotation_text=f"Best Return: {best_return:.2%}", annotation_position="top left")
                            fig_simulated_returns.add_vline(x=worst_return, line_dash="dash", line_color="red", annotation_text=f"Worst Return: {worst_return:.2%}", annotation_position="top left")
                            
                            st.plotly_chart(fig_simulated_returns, use_container_width=True)
                            
                            # Additional Visualization
                            st.subheader("Cumulative Returns from Simulations")
                            cum_returns_sim = 1 + simulated_returns
                            fig_cum_returns = px.histogram(cum_returns_sim, nbins=50, title='Cumulative Returns from Monte Carlo Simulations', labels={'value': 'Cumulative Returns', 'count': 'Frequency'})
                            st.plotly_chart(fig_cum_returns, use_container_width=True)

                            # More Visualizations
                            # Probability of Achieving Target Return
                            st.subheader("Probability of Achieving Target Return")
                            target_return = st.number_input("Target Return (%)", value=5.0) / 100
                            probability = np.mean(simulated_returns >= target_return)
                            st.write(f"Probability of achieving at least {target_return*100:.2f}% annual return: {probability*100:.2f}%")

                            fig_prob = px.bar(x=["Achieved", "Not Achieved"], y=[probability*100, (1 - probability)*100], 
                                             title='Probability of Achieving Target Return', 
                                             labels={'x': 'Outcome', 'y': 'Probability (%)'}, 
                                             text_auto=True)
                            st.plotly_chart(fig_prob, use_container_width=True)
                            
                            # Cumulative Distribution Function (CDF) of Simulated Returns
                            st.subheader("📈 Cumulative Distribution Function (CDF) of Simulated Returns")
                            sorted_returns = np.sort(simulated_returns)
                            cdf = np.linspace(0, 1, len(sorted_returns))
                            fig_cdf = px.line(x=sorted_returns, y=cdf, title='CDF of Simulated Annual Returns',
                                             labels={'x': 'Simulated Annual Return', 'y': 'Cumulative Probability'})
                            st.plotly_chart(fig_cdf, use_container_width=True)

                            # Time Series Plot of Simulated Returns (First 10 Simulations)
                            st.subheader("📈 Time Series of Simulated Annual Returns (First 10 Simulations)")
                            fig_time_series = px.line(pd.DataFrame(simulated_returns[:10], columns=['Simulated Return']),
                                                     title='Time Series of Simulated Annual Returns',
                                                     labels={'index': 'Simulation Number', 'Simulated Return': 'Annual Return'})
                            st.plotly_chart(fig_time_series, use_container_width=True)

                            # Interpretation
                            st.markdown("**Interpretation:** This analysis provides the probability of your portfolio achieving or exceeding the specified target return based on the simulated outcomes. A higher probability indicates a greater likelihood of meeting your investment goals.")

                            # Store results
                            st.session_state.backtest_results = {
                                'simulated_returns': simulated_returns
                            }
                            st.success("Monte Carlo simulations completed!")
    else:
        st.warning("Please add at least one portfolio to run Monte Carlo simulations.")

elif step == "Risk Analysis":
    st.title("⚠️ Risk Analysis")
    if st.session_state.portfolios:
        portfolio_names = [p['name'] for p in st.session_state.portfolios]
        selected_portfolio = st.selectbox("Select Portfolio for Risk Analysis", portfolio_names)
        if selected_portfolio:
            portfolio = next(p for p in st.session_state.portfolios if p['name'] == selected_portfolio)
            price_data = download_data(
                portfolio['selected'],
                portfolio['start_date'],
                portfolio['end_date']
            )
            if price_data.empty:
                st.error("No price data available for the selected portfolio.")
            else:
                available_selected = [ticker for ticker in portfolio['selected'] if ticker in price_data.columns]
                missing_selected = list(set(portfolio['selected']).difference(set(available_selected)))
                if missing_selected:
                    st.warning(f"Excluded tickers with no data: {', '.join(missing_selected)}")
                if not available_selected:
                    st.error(
        "No selected tickers have available data for the chosen date range. "
        "Please ensure that the ticker symbols are correct and data is available for the specified period."
    )
                else:
                    returns = price_data[available_selected].pct_change().dropna()
                    confidence_level = st.slider("Confidence Level", min_value=0.01, max_value=0.99, value=0.95)
                    if st.button("Calculate Risk Metrics"):
                        with st.spinner("Calculating risk metrics..."):
                            # Calculate portfolio returns as weighted average
                            weights = np.array(portfolio['allocations']) / 100
                            portfolio_returns = returns.dot(weights) if isinstance(returns, pd.DataFrame) else returns
                            var = calculate_var(portfolio_returns, confidence_level)
                            cvar = calculate_cvar(portfolio_returns, confidence_level)
                            st.write(f"Value at Risk (VaR) at {confidence_level:.0%} confidence level: {var:.2f}")
                            st.write(f"Conditional Value at Risk (CVaR) at {confidence_level:.0%} confidence level: {cvar:.2f}")
                            st.markdown("**Interpretation:** This table summarizes the key risk metrics of your portfolio. VaR indicates the maximum expected loss at the specified confidence level, CVaR provides the average loss beyond the VaR, Annualized Volatility measures the portfolio's risk, and Maximum Drawdown represents the largest peak-to-trough decline.")

                            # Risk Metrics Summary Table
                            st.markdown("### 📈 Risk Metrics Summary")
                            cum_returns = (1 + portfolio_returns).cumprod()
                            risk_metrics = {
                                'Value at Risk (VaR)': f"{var:.2f}",
                                'Conditional Value at Risk (CVaR)': f"{cvar:.2f}",
                                'Annualized Volatility (%)': f"{portfolio_returns.std() * 100 * np.sqrt(252):.2f}%",
                                'Maximum Drawdown (%)': f"{drawdown(cum_returns) * 100:.2f}%"
                            }
                            risk_df = pd.DataFrame(list(risk_metrics.items()), columns=['Metric', 'Value'])
                            st.table(risk_df)
                            st.markdown("""
                            **Tooltip:**
                            - **VaR:** Measures the worst expected loss under normal market conditions over a specific time period at a given confidence level.
                            - **CVaR:** Provides the average loss exceeding the VaR, offering insight into tail risk.
                            - **Annualized Volatility:** Indicates the degree of variation in portfolio returns, representing risk.
                            - **Maximum Drawdown:** Shows the largest peak-to-trough decline, indicating potential risk exposure.
                            """)
                            # Real-Time Risk Warning
                            high_var_threshold = -0.10  # Example: VaR worse than -10%
                            if var <= high_var_threshold:
                                st.warning(f"⚠️ High VaR! Your portfolio has a {confidence_level*100:.0f}% chance of losing over {abs(var)*100:.2f}% in the next year.")
                            else:
                                st.success(f"✅ VaR is within acceptable limits: {var*100:.2f}%")
                            # Additional Visualizations
                            st.subheader("Value at Risk (VaR) Plot")
                            fig_var = px.line(portfolio_returns.cumsum(), title='Value at Risk (VaR) Over Time', labels={'value': 'Cumulative Returns', 'index': 'Date'})
                            fig_var.add_hline(y=var, line_dash="dash", line_color="red", annotation_text=f"VaR: {var:.2f}", annotation_position="bottom right")
                            st.plotly_chart(fig_var, use_container_width=True)
        
                            st.subheader("Conditional Value at Risk (CVaR) Plot")
                            fig_cvar = px.line(portfolio_returns.cumsum(), title='Conditional Value at Risk (CVaR) Over Time', labels={'value': 'Cumulative Returns', 'index': 'Date'})
                            fig_cvar.add_hline(y=cvar, line_dash="dash", line_color="blue", annotation_text=f"CVaR: {cvar:.2f}", annotation_position="bottom right")
                            st.plotly_chart(fig_cvar, use_container_width=True)
                            
                            # VaR and CVaR Distribution Plot
                            st.subheader("📊 VaR and CVaR Distribution")
                            fig_var_cvar_dist = px.histogram(
                                portfolio_returns, 
                                nbins=50, 
                                title='Returns Distribution with VaR and CVaR',
                                labels={'value': 'Returns', 'count': 'Frequency'}
                            )
                            fig_var_cvar_dist.add_vline(x=var, line_dash="dash", line_color="red", annotation_text=f"VaR: {var:.2f}", annotation_position="top left")
                            fig_var_cvar_dist.add_vline(x=cvar, line_dash="dash", line_color="blue", annotation_text=f"CVaR: {cvar:.2f}", annotation_position="top left")
                            st.plotly_chart(fig_var_cvar_dist, use_container_width=True)
                            st.markdown("**Interpretation:** This histogram displays the distribution of portfolio returns with vertical lines indicating the VaR and CVaR levels. VaR represents the threshold below which a certain percentage of losses occur, while CVaR provides the average loss beyond the VaR threshold.")
        
                            # Increase Readability
                            st.markdown("### 📈 Risk Metrics Summary")
                            risk_metrics = {
                                'Value at Risk (VaR)': f"{var:.2f}",
                                'Conditional Value at Risk (CVaR)': f"{cvar:.2f}"
                            }
                            risk_df = pd.DataFrame(list(risk_metrics.items()), columns=['Metric', 'Value'])
                            st.table(risk_df)
                            st.markdown("### 📝 Risk Summary")
                            if var <= high_var_threshold:
                                st.write(f"Your portfolio is **highly risky**, with a maximum potential loss of **{abs(var)*100:.2f}%** in the worst case.")
                            elif var <= -0.05:
                                st.write(f"Your portfolio is **moderately risky**, with a maximum potential loss of **{abs(var)*100:.2f}%** in the worst case.")
                            else:
                                st.write(f"Your portfolio is **low risk**, with a maximum potential loss of **{abs(var)*100:.2f}%** in the worst case.")
                            # Risk Analysis - Risk vs Return Scatter Plot
                            st.subheader("📊 Risk vs Return Scatter Plot")
                            risk = returns.std() * np.sqrt(252)
                            ret = returns.mean() * 252

                            if isinstance(risk, (float, int)):
                                scatter_df = pd.DataFrame({
                                    'Risk': [risk],
                                    'Return': [ret]
                                })
                            else:
                                scatter_df = pd.DataFrame({
                                    'Risk': risk,
                                    'Return': ret
                                })

                            if isinstance(returns, pd.DataFrame) and returns.shape[1] > 1:
                                scatter_df['Asset'] = available_selected

                            fig_risk_return = px.scatter(scatter_df, x='Risk', y='Return',
                                                         title='Risk vs Return Scatter Plot',
                                                         labels={'Risk': 'Risk (Std Dev)', 'Return': 'Return'},
                                                         hover_data=['Asset'] if isinstance(returns, pd.DataFrame) and returns.shape[1] > 1 else [])
                            st.plotly_chart(fig_risk_return, use_container_width=True)

    else:
        st.warning("Please add at least one portfolio to run risk analysis.")