# components/run_backtest.py
import streamlit as st
import pandas as pd
import numpy as np
import plotly.express as px

# Importing all necessary functions from utils, metrics, and plots
from utils import (
    download_data, 
    generate_recommendations, 
    backtest,  
    fetch_fama_french_factors, 
)

from metrics import (
    calculate_portfolio_metrics, 
    calculate_final_score, 
    calculate_risk_factor_attribution, 
    get_drawdown_details
)

from plots import (
    plot_growth_comparison, 
    plot_drawdown_comparison, 
    plot_cagr_over_time, 
    plot_allocation_pie, 
    plot_correlation_heatmap, 
    plot_risk_return_attribution, 
    plot_rolling_metrics
)

def run_backtest_ui():
    # Initialize portfolios in session state if not already initialized
    if 'portfolios' not in st.session_state:
        st.session_state.portfolios = []
    
    if st.session_state.portfolios:
        with st.form("compare_form"):
            st.subheader("🔍 Select Portfolios to Compare")
            comparison_type = st.selectbox(
                "Comparison Type",
                ["Portfolio vs Benchmark", "Portfolio vs Portfolio"]
            )
            portfolio_names = [p['name'] for p in st.session_state.portfolios]
            if comparison_type == "Portfolio vs Benchmark":
                portfolio1 = st.selectbox("Select Portfolio", portfolio_names)
                portfolio2 = None
            else:
                portfolio1 = st.selectbox("Select First Portfolio", portfolio_names, key="p1")
                portfolio2 = st.selectbox("Select Second Portfolio", portfolio_names, key="p2")
                if portfolio1 == portfolio2:
                    st.warning("Please select two different portfolios for comparison.")
            
            st.subheader("⚙️ Visualizations Settings")
            selected_time_frames = st.multiselect(
                "Select Time Frames for CAGR",
                options=['Weekly', 'Monthly', 'Quarterly', 'Annually'],
                default=['Annually'],
                key="selected_time_frames",
                help="Choose one or more time frames to view CAGR over different horizons."
            )
            selected_rolling_periods = st.multiselect(
                "Select Rolling Periods (Days)",
                options=[30, 90, 180, 252],
                default=[252],
                key="selected_rolling_periods",
                help="Choose one or more periods to view rolling metrics."
            )

            run = st.form_submit_button("Run Backtest")
        
        if run:
            with st.spinner("Running backtest..."):
                if comparison_type == "Portfolio vs Benchmark":
                    # Handle Portfolio vs Benchmark
                    try:
                        portfolio = next(p for p in st.session_state.portfolios if p['name'] == portfolio1)
                    except StopIteration:
                        st.error("Selected portfolio not found.")
                        return

                    price_data = download_data(
                        portfolio['selected'],
                        portfolio['start_date'],
                        portfolio['end_date']
                    )
                    if price_data.empty:
                        st.error("No price data available for the selected portfolio.")
                    else:
                        available_selected = [ticker for ticker in portfolio['selected'] if ticker in price_data.columns]
                        missing_selected = list(set(portfolio['selected']) - set(available_selected))
                        if missing_selected:
                            st.warning(f"Excluded tickers with no data: {', '.join(missing_selected)}")
                        if not available_selected:
                            st.error(
                                "No selected tickers have available data for the chosen date range. "
                                "Please ensure that the ticker symbols are correct and data is available for the specified period."
                            )
                        else:
                            allocations = [portfolio.get('allocations', [0.0]*len(portfolio.get('selected', [])))[i]
                                          for i, ticker in enumerate(portfolio.get('selected', [])) if ticker in available_selected]
                            if not np.isclose(sum(allocations), 100.0, atol=1e-4):
                                st.warning("Allocations do not sum to 100%. Adjusting allocations proportionally.")
                                allocations = [a / sum(allocations) * 100 for a in allocations]
                            weights = np.array(allocations) / 100
                            returns, cum_returns = backtest(weights, price_data[available_selected],
                                                            portfolio['rebalance_freq'], portfolio['broker_fee'], debug=False)
                            
                            # **Fix: Ensure returns have a DateTime index with matching length**
                            if not isinstance(returns.index, pd.DatetimeIndex):
                                try:
                                    returns.index = pd.to_datetime(price_data.index[:len(returns)])
                                    cum_returns.index = pd.to_datetime(price_data.index[:len(cum_returns)])
                                except Exception as e:
                                    st.error(f"Error converting returns index to DateTimeIndex: {e}")
                                    return
                            
                            if cum_returns.empty:
                                st.error("Cumulative returns are empty. Check the data and allocations.")
                            else:
                                benchmark_symbol = portfolio['benchmark_symbol']
                                benchmark_data = download_data([benchmark_symbol], portfolio['start_date'], portfolio['end_date'])

                                if benchmark_symbol not in benchmark_data.columns:
                                    st.error(f"The selected benchmark symbol '{benchmark_symbol}' does not have available data for the chosen period.")
                                    st.stop()
                                else:
                                    st.success(f"Benchmark '{benchmark_symbol}' data successfully downloaded and will be used for comparison.")
                                    if not benchmark_data.empty:
                                        benchmark_returns = benchmark_data[benchmark_symbol].pct_change().dropna()
                                        benchmark_returns = benchmark_returns.reindex(returns.index, method='ffill').dropna()
                                        common_index = returns.index.intersection(benchmark_returns.index)
                                        if common_index.empty:
                                            st.error("No overlapping dates between portfolio returns and benchmark returns after alignment.")
                                            st.stop()
                                        returns = returns.loc[common_index]
                                        cum_returns = cum_returns.loc[common_index]
                                        # Align the data using an inner join
                                        aligned_data = pd.merge(returns.to_frame('Portfolio'), benchmark_returns.to_frame('Benchmark'), left_index=True, right_index=True, how='inner')

                                        # Check that we have data after merging
                                        if aligned_data.empty:
                                            st.error("No overlapping dates between portfolio returns and benchmark returns after alignment.")
                                            st.stop()

                                        # Assign aligned returns
                                        returns = aligned_data['Portfolio'].rename("Portfolio_Returns")
                                        benchmark_returns = aligned_data['Benchmark']
                                        benchmark_cum_returns = (1 + benchmark_returns).cumprod()
                                        benchmark_metrics = calculate_portfolio_metrics(
                                            benchmark_returns, benchmark_cum_returns, portfolio['rf_rate'], None
                                        )
                                    else:
                                        benchmark_returns = None
                                        benchmark_cum_returns = pd.Series(dtype=float)
                                        benchmark_metrics = {
                                            'Start Balance': "N/A",
                                            'End Balance': "N/A",
                                            'Annualized Return (CAGR)': "N/A",
                                            'Best Year': "N/A",
                                            'Worst Year': "N/A",
                                            'Arithmetic Mean (Monthly)': "N/A",
                                            'Arithmetic Mean (Annualized)': "N/A",
                                            'Geometric Mean (Monthly)': "N/A",
                                            'Geometric Mean (Annualized)': "N/A",
                                            'Standard Deviation (Monthly)': "N/A",
                                            'Standard Deviation (Annualized)': "N/A",
                                            'Downside Deviation (Monthly)': "N/A",
                                            'Maximum Drawdown': "N/A",
                                            'Sharpe Ratio': "N/A",
                                            'Sortino Ratio': "N/A",
                                            'Gain/Loss Ratio': "N/A",
                                            'Skewness': "N/A",
                                            'Excess Kurtosis': "N/A",
                                            'Safe Withdrawal Rate': "N/A",
                                            'Perpetual Withdrawal Rate': "N/A",
                                            'Positive Periods': "N/A",
                                            'Benchmark Correlation': "N/A",
                                            'Beta': "N/A",
                                            'Alpha (annualized)': "N/A",
                                            'R2': "N/A",
                                            'Treynor Ratio': "N/A",
                                            'Calmar Ratio': "N/A",
                                            'Modigliani–Modigliani Measure': "N/A",
                                            'Information Ratio': "N/A",
                                            'Tracking Error': "N/A",
                                            'Active Return': "N/A",
                                            'Upside Capture Ratio': "N/A",
                                            'Downside Capture Ratio': "N/A"
                                        }

                                if not cum_returns.empty:
                                    portfolio_metrics = calculate_portfolio_metrics(
                                        returns, cum_returns, portfolio['rf_rate'], benchmark_returns
                                    )

                                    # Dashboard Header with Key Stats
                                    with st.container():
                                        st.markdown("### 🔑 Key Metrics")
                                        key_metrics = {
                                            'Annualized Return (CAGR)': portfolio_metrics.get('Annualized Return (CAGR)', "N/A"),
                                            'Sharpe Ratio': portfolio_metrics.get('Sharpe Ratio', "N/A"),
                                            'Maximum Drawdown': portfolio_metrics.get('Maximum Drawdown', "N/A")
                                        }
                                        cols = st.columns(len(key_metrics))
                                        for col, (metric, value) in zip(cols, key_metrics.items()):
                                            with col:
                                                st.metric(label=metric, value=value)

                                    # Create Tabs for Organized Sections
                                    tabs = st.tabs(["Overview", "Performance Statistics", "Advanced Metrics", "Drawdowns", "Visualizations"])

                                    with tabs[0]:
                                        st.header("📈 Portfolio Performance Overview")
                                        st.write(f"**Portfolio Name:** {portfolio['name']}")
                                        st.write(f"**Start Date:** {portfolio['start_date'].strftime('%Y-%m-%d')}")
                                        st.write(f"**End Date:** {portfolio['end_date'].strftime('%Y-%m-%d')}")
                                        st.write(f"**Benchmark:** {portfolio['benchmark_symbol']}")

                                    with tabs[1]:
                                        st.header("📊 Performance Statistics")
                                        
                                        with st.container():
                                            st.subheader("🛠️ Basic Metrics")
                                            performance_data_stats = {
                                                'Metric': [
                                                    'Start Balance',
                                                    'End Balance',
                                                    'Annualized Return (CAGR)',
                                                    'Standard Deviation (Annualized)',
                                                    'Best Year',
                                                    'Worst Year',
                                                    'Maximum Drawdown',
                                                    'Sharpe Ratio',
                                                    'Sortino Ratio',
                                                    'Benchmark Correlation'
                                                ],
                                                'Portfolio': [
                                                    portfolio_metrics['Start Balance'],
                                                    portfolio_metrics['End Balance'],
                                                    portfolio_metrics['Annualized Return (CAGR)'],
                                                    portfolio_metrics['Standard Deviation (Annualized)'],
                                                    portfolio_metrics['Best Year'],
                                                    portfolio_metrics['Worst Year'],
                                                    portfolio_metrics['Maximum Drawdown'],
                                                    portfolio_metrics['Sharpe Ratio'],
                                                    portfolio_metrics['Sortino Ratio'],
                                                    portfolio_metrics['Benchmark Correlation']
                                                ],
                                                'Benchmark': [
                                                    benchmark_metrics['Start Balance'],
                                                    benchmark_metrics['End Balance'],
                                                    benchmark_metrics['Annualized Return (CAGR)'],
                                                    benchmark_metrics['Standard Deviation (Annualized)'],
                                                    benchmark_metrics['Best Year'],
                                                    benchmark_metrics['Worst Year'],
                                                    benchmark_metrics['Maximum Drawdown'],
                                                    benchmark_metrics['Sharpe Ratio'],
                                                    benchmark_metrics['Sortino Ratio'],
                                                    benchmark_metrics['Benchmark Correlation']
                                                ]
                                            }
                                            performance_df_stats = pd.DataFrame(performance_data_stats).set_index('Metric')
                                            st.table(performance_df_stats)

                                    with tabs[2]:
                                        st.header("📋 Detail Comparisons")
                                        with st.container():
                                            st.subheader("🧮 Advanced Metrics")
                                            performance_data_advanced = {
                                                'Metric': list(portfolio_metrics.keys()),
                                                'Portfolio': list(portfolio_metrics.values()),
                                                'Benchmark': list(benchmark_metrics.values())
                                            }
                                            performance_df_advanced = pd.DataFrame(performance_data_advanced).set_index('Metric')
    
                                            # Replace NaN with empty strings for better visualization
                                            performance_df_advanced = performance_df_advanced.replace(np.nan, "")
                                            # Drop rows where both Portfolio and Benchmark values are NaN
                                            performance_df_advanced_clean = performance_df_advanced.dropna(how='all')
    
                                            st.table(performance_df_advanced_clean)
                                            st.markdown("---")
                                            st.write("**Note:** Some metrics in the benchmark column are empty because they are portfolio-specific measurements that can't be calculated for the benchmark alone. These include:\n\n"
                         "1. Alpha (annualized)\n"
                         "2. Information Ratio\n"
                         "3. Tracking Error\n"
                         "4. Active Return\n"
                         "5. Upside/Downside Capture Ratios\n\n"
                         "These metrics specifically measure how a portfolio performs relative to its benchmark, so they only make sense when calculated for the portfolio itself. For example, tracking error measures how closely a portfolio follows its benchmark, which isn't applicable to the benchmark itself. Similarly, alpha measures excess return relative to the benchmark, which wouldn't be meaningful to calculate for the benchmark itself.")
                                        st.subheader("🔍 Risk Factor Attribution Analysis")
                                        
                                        # Ensure 'returns' is defined and is a pd.Series
                                        if 'returns' not in locals():
                                            st.error("Portfolio returns data ('returns') is not defined.")
                                        else:
                                            # Define date range based on portfolio returns index
                                            start_date = returns.index.min().strftime('%Y-%m-%d')
                                            end_date = returns.index.max().strftime('%Y-%m-%d')
                                            # Fetch actual factor data
                                            factors = fetch_fama_french_factors(start_date, end_date)
                                            # Ensure factors are fetched
                                            if factors.empty:
                                                st.write("Failed to retrieve factor data.")       
                                            else:
                                                if isinstance(returns.index, pd.DatetimeIndex):
                                                    if returns.index.freq is None:
                                                        inferred_freq = pd.infer_freq(returns.index)
                                                        if inferred_freq is None:
                                                            inferred_freq = 'D'  # Set default frequency to daily
                                                    else:
                                                        inferred_freq = returns.index.freq
                                                    
                                                    try:
                                                        factors = factors.asfreq(inferred_freq, method='ffill')
                                                    except TypeError as e:
                                                        st.error(f"Error setting frequency for factors: {e}")
                                                        st.stop()
                                                else:
                                                    st.error("Returns index is not a DatetimeIndex.")
                                                    st.stop()
                                                # Calculate Attribution with aligned dates
                                                attribution, error = calculate_risk_factor_attribution(returns, factors)
                                                
                                                if error:
                                                    st.error(error)
                                                    st.write("Risk factor attribution analysis is unavailable.")
                                                elif attribution.empty:
                                                    st.write("Risk factor attribution analysis is unavailable.")
                                                else:
                                                    st.write("Risk Factor Attribution:")
                                                    st.table(attribution)
                                                    # Optional: Visualize Attribution
                                                    fig_attribution = px.bar(
                                                        attribution,
                                                        x='Factor',
                                                        y='Contribution (%)',
                                                        title='Risk Factor Attribution',
                                                        labels={'Contribution (%)': 'Contribution (%)'},
                                                        template='plotly_dark'
                                                    )
                                                    st.plotly_chart(fig_attribution, use_container_width=True)

                                    with tabs[3]:
                                        st.header("📉 Detailed Drawdowns")
                                        st.markdown("### 📈 Drawdowns for Portfolio")
                                        portfolio_drawdowns = get_drawdown_details(cum_returns)
                                        if portfolio_drawdowns:
                                            portfolio_drawdowns_df = pd.DataFrame(portfolio_drawdowns)
                                            st.table(portfolio_drawdowns_df)
                                        else:
                                            st.write("No drawdowns detected for the portfolio.")

                                        if benchmark_returns is not None and not benchmark_returns.empty:
                                            st.markdown("### 📈 Drawdowns for Benchmark")
                                            benchmark_drawdowns = get_drawdown_details(benchmark_cum_returns)
                                            if benchmark_drawdowns:
                                                benchmark_drawdowns_df = pd.DataFrame(benchmark_drawdowns)
                                                st.table(benchmark_drawdowns_df)
                                            else:
                                                st.write("No drawdowns detected for the benchmark.")
                                        else:
                                            st.write("Benchmark data not available for drawdown analysis.")

                                    with tabs[4]:
                                        st.header("📊 Visualizations")
                                        if not cum_returns.empty and not benchmark_cum_returns.empty:
                                            plot_growth_comparison(cum_returns, benchmark_cum_returns, labels=["Portfolio", "Benchmark"])
                                        else:
                                            st.warning("Insufficient data to display Growth Comparison. Ensure both portfolio and benchmark have data.")
                                        
                                        portfolio_drawdown_series = (cum_returns / cum_returns.expanding().max() - 1) * 100
                                        benchmark_drawdown_series = (benchmark_cum_returns / benchmark_cum_returns.expanding().max() - 1) * 100
                                        plot_drawdown_comparison(portfolio_drawdown_series, benchmark_drawdown_series, labels=["Portfolio", "Benchmark"])

                                        st.subheader("📈 Compound Annual Growth Rate (CAGR) Over Time")
                                        if selected_time_frames:
                                            plot_cagr_over_time(cum_returns, time_frames=selected_time_frames, label="Portfolio")
                                        else:
                                            st.warning("Please select at least one time frame for CAGR.")

                                        # Box Plot for Returns Distribution
                                        st.subheader("📦 Returns Distribution Box Plot")
                                        if isinstance(returns, pd.DataFrame):
                                            melted_returns = returns.reset_index().melt(id_vars='Date', var_name='Asset', value_name='Return')
                                            fig_box_plot = px.box(
                                                melted_returns,
                                                x='Asset',
                                                y='Return',
                                                title='Returns Distribution Box Plot',
                                                labels={'Return': 'Returns', 'Asset': 'Asset'},
                                                points='all',
                                                hover_data=['Return'],
                                                template='plotly_dark',
                                                color='Asset',
                                                color_discrete_sequence=px.colors.qualitative.Set3
                                            )
                                            st.plotly_chart(fig_box_plot, use_container_width=True)
                                            st.markdown("**Interpretation:** The box plot visualizes the distribution of returns for each asset in the portfolio. The boxes represent the interquartile range (IQR), the line inside the box indicates the median, and the whiskers show the range of the data. Outliers are displayed as individual points.")
                                        elif isinstance(returns, pd.Series):
                                            fig_box_plot = px.box(
                                                returns.to_frame(name='Return'),
                                                y='Return',
                                                title='Returns Distribution Box Plot',
                                                labels={'Return': 'Returns'},
                                                points='all',
                                                hover_data=['Return'],
                                                template='plotly_dark'
                                            )
                                            st.plotly_chart(fig_box_plot, use_container_width=True)
                                            st.markdown("**Interpretation:** The box plot visualizes the distribution of returns for the portfolio. The box represents the interquartile range (IQR), the line inside the box indicates the median, and the whiskers show the range of the data. Outliers are displayed as individual points.")
                                        else:
                                            st.warning("Returns data is neither a DataFrame nor a Series.")

                                        # Heatmap of Correlations Between Assets
                                        st.subheader("🔥 Correlation Heatmap of Assets")
                                        plot_correlation_heatmap(returns)

                                        # Add Risk-Return Attribution Analysis
                                        st.subheader("🔍 Risk-Return Attribution Analysis")
                                        plot_risk_return_attribution(returns, weights, label='Portfolio')

                                        # Cumulative Returns Heatmap
                                        st.subheader("🔥 Cumulative Returns Heatmap")
                                        try:
                                            cum_returns_normalized = cum_returns / cum_returns.max()
                                            fig_cum_heatmap = px.imshow(
                                                cum_returns_normalized.to_frame().T,
                                                labels=dict(x="Date", y="Portfolio", color="Normalized Cumulative Return"),
                                                title="Cumulative Returns Heatmap",
                                                aspect="auto",
                                                color_continuous_scale='Viridis',
                                                template='plotly_dark'
                                            )
                                            st.plotly_chart(fig_cum_heatmap, use_container_width=True)
                                            st.markdown("**Interpretation:** The heatmap visualizes the normalized cumulative returns over time, allowing for an intuitive comparison of portfolio performance across different periods.")
                                        except Exception as e:
                                            st.error(f"Error plotting cumulative returns heatmap: {e}")

                                        # Portfolio Allocation Pie Chart
                                        st.subheader("🥧 Portfolio Allocation Pie Chart")
                                        plot_allocation_pie(weights, available_selected, title="Portfolio Allocation", hover_info="percent+name")

                                        # Rolling Metrics Chart with User-Defined Periods
                                        st.subheader("📉 Rolling Metrics")
                                        if selected_rolling_periods:
                                            plot_rolling_metrics(returns, windows=selected_rolling_periods, label='Portfolio')
                                        else:
                                            st.warning("Please select at least one rolling period to display metrics.")

                                        # Recommendations
                                        st.markdown("### 💡 Recommendations")
                                        recommendations = generate_recommendations(
                                            allocations=[alloc for alloc in portfolio['allocations'] if alloc > 0],
                                            available_selected=[ticker for ticker in available_selected if portfolio['allocations'][available_selected.index(ticker)] > 0]
                                        )
                                        
                                        if recommendations:
                                            for rec in recommendations:
                                                st.write(rec)
                                        else:
                                            st.success("Your portfolio allocations are well-balanced!")
                                            
                                        # Final Score
                                        st.markdown("### 📝 Final Score")
                                        portfolio_score = calculate_final_score(portfolio_metrics, benchmark_metrics)
                                        benchmark_score = calculate_final_score(benchmark_metrics, portfolio_metrics)
                                        st.write(f"**Portfolio:** {portfolio_score:.2f} / 100")
                                        st.write(f"**Benchmark:** {benchmark_score:.2f} / 100")

                                        # Store results
                                        st.session_state.backtest_results = {
                                            'returns': returns,
                                            'cum_returns': cum_returns,
                                            'weights_a': weights,
                                            'price_data': price_data[available_selected],
                                            'metrics': portfolio_metrics,
                                            'benchmark_metrics': benchmark_metrics,
                                            'performance_df_stats': performance_df_stats,
                                            'performance_df_advanced': performance_df_advanced_clean,  # Corrected to use the cleaned DataFrame
                                            'benchmark_cum_returns': benchmark_cum_returns
                                        }
                elif comparison_type == "Portfolio vs Portfolio" and portfolio1 and portfolio2 and portfolio1 != portfolio2:
                    # Handle Portfolio vs Portfolio
                    try:
                        portfolio_a = next(p for p in st.session_state.portfolios if p['name'] == portfolio1)
                        portfolio_b = next(p for p in st.session_state.portfolios if p['name'] == portfolio2)
                    except StopIteration:
                        st.error("One or both selected portfolios were not found.")
                        return

                    # Backtest Portfolio A
                    price_data_a = download_data(
                        portfolio_a['selected'],
                        portfolio_a['start_date'],
                        portfolio_a['end_date']
                    )
                    # Backtest Portfolio B
                    price_data_b = download_data(
                        portfolio_b['selected'],
                        portfolio_b['start_date'],
                        portfolio_b['end_date']
                    )

                    if price_data_a.empty or price_data_b.empty:
                        st.error("No price data available for one or both selected portfolios.")
                    else:
                        # Process Portfolio A
                        available_a = [ticker for ticker in portfolio_a['selected'] if ticker in price_data_a.columns]
                        missing_a = list(set(portfolio_a['selected']) - set(available_a))
                        if missing_a:
                            st.warning(f"Excluded tickers from {portfolio_a['name']} with no data: {', '.join(missing_a)}")
                        if not available_a:
                            st.error(f"No selected tickers have available data for the portfolio '{portfolio_a['name']}'.")
                            return
                        allocations_a = [portfolio_a.get('allocations', [0.0]*len(portfolio_a.get('selected', [])))[i]
                                         for i, ticker in enumerate(portfolio_a.get('selected', [])) if ticker in available_a]
                        if not np.isclose(sum(allocations_a), 100.0, atol=1e-4):
                            st.warning(f"Allocations for {portfolio_a['name']} do not sum to 100%. Adjusting allocations proportionally.")
                            allocations_a = [a / sum(allocations_a) * 100 for a in allocations_a]
                        weights_a = np.array(allocations_a) / 100
                        returns_a, cum_returns_a = backtest(weights_a, price_data_a[available_a],
                                                           portfolio_a['rebalance_freq'], portfolio_a['broker_fee'], debug=False)
                        
                        # Process Portfolio B
                        available_b = [ticker for ticker in portfolio_b['selected'] if ticker in price_data_b.columns]
                        missing_b = list(set(portfolio_b['selected']) - set(available_b))
                        if missing_b:
                            st.warning(f"Excluded tickers from {portfolio_b['name']} with no data: {', '.join(missing_b)}")
                        if not available_b:
                            st.error(f"No selected tickers have available data for the portfolio '{portfolio_b['name']}'.")
                            return
                        allocations_b = [portfolio_b.get('allocations', [0.0]*len(portfolio_b.get('selected', [])))[i]
                                         for i, ticker in enumerate(portfolio_b.get('selected', [])) if ticker in available_b]
                        if not np.isclose(sum(allocations_b), 100.0, atol=1e-4):
                            st.warning(f"Allocations for {portfolio_b['name']} do not sum to 100%. Adjusting allocations proportionally.")
                            allocations_b = [a / sum(allocations_b) * 100 for a in allocations_b]
                        weights_b = np.array(allocations_b) / 100
                        returns_b, cum_returns_b = backtest(weights_b, price_data_b[available_b],
                                                           portfolio_b['rebalance_freq'], portfolio_b['broker_fee'], debug=False)
                        
                        # **Fix: Ensure returns have a DateTime index with matching length**
                        try:
                            returns_a.index = pd.to_datetime(price_data_a.index[:len(returns_a)])
                            cum_returns_a.index = pd.to_datetime(price_data_a.index[:len(cum_returns_a)])
                            returns_b.index = pd.to_datetime(price_data_b.index[:len(returns_b)])
                            cum_returns_b.index = pd.to_datetime(price_data_b.index[:len(cum_returns_b)])
                        except Exception as e:
                            st.error(f"Error converting returns index to DateTimeIndex: {e}")
                            return
                        
                        if cum_returns_a.empty or cum_returns_b.empty:
                            st.error("Cumulative returns are empty for one or both portfolios. Check the data and allocations.")
                            return
                        
                        # Align the data for comparison
                        common_dates = cum_returns_a.index.intersection(cum_returns_b.index)
                        if common_dates.empty:
                            st.error("No overlapping dates between the two portfolios after alignment.")
                            return
                        cum_returns_a = cum_returns_a.loc[common_dates]
                        cum_returns_b = cum_returns_b.loc[common_dates]
                        returns_a = returns_a.loc[common_dates]
                        returns_b = returns_b.loc[common_dates]
                        
                        # Calculate Metrics
                        metrics_a = calculate_portfolio_metrics(
                            returns_a, cum_returns_a, portfolio_a['rf_rate'], None
                        )
                        metrics_b = calculate_portfolio_metrics(
                            returns_b, cum_returns_b, portfolio_b['rf_rate'], None
                        )
                        
                        # Dashboard Header with Key Stats
                        with st.container():
                            st.markdown("### 🔑 Key Metrics")
                            key_metrics = {
                                'Annualized Return (CAGR)': ('CAGR', metrics_a.get('Annualized Return (CAGR)', "N/A"), metrics_b.get('Annualized Return (CAGR)', "N/A")),
                                'Sharpe Ratio': ('Sharpe Ratio', metrics_a.get('Sharpe Ratio', "N/A"), metrics_b.get('Sharpe Ratio', "N/A")),
                                'Maximum Drawdown': ('Maximum Drawdown', metrics_a.get('Maximum Drawdown', "N/A"), metrics_b.get('Maximum Drawdown', "N/A"))
                            }
                            columns = st.columns(len(key_metrics))
                            for (metric, (label, value_a, value_b)) in key_metrics.items():
                                with columns[list(key_metrics.keys()).index(metric)]:
                                    st.metric(label=f"{label} ({portfolio_a['name']})", value=value_a)
                                    st.metric(label=f"{label} ({portfolio_b['name']})", value=value_b)

                        # Create Tabs for Organized Sections
                        tabs = st.tabs(["Overview", "Performance Statistics", "Advanced Metrics", "Drawdowns", "Visualizations"])

                        with tabs[0]:
                            st.header("📈 Portfolios Performance Overview")
                            # Determine the aligned start and end dates
                            aligned_start_date = common_dates.min().strftime('%Y-%m-%d')
                            aligned_end_date = common_dates.max().strftime('%Y-%m-%d')
                            
                            st.write(f"**Aligned Date Range:** {aligned_start_date} to {aligned_end_date}")
                            st.write(f"**Portfolio A:** {portfolio_a['name']}")
                            st.write(f"**Original Start Date:** {portfolio_a['start_date'].strftime('%Y-%m-%d')}")
                            st.write(f"**Portfolio B:** {portfolio_b['name']}")
                            st.write(f"**Original Start Date:** {portfolio_b['start_date'].strftime('%Y-%m-%d')}")

                        with tabs[1]:
                            st.header("📊 Performance Statistics")
                            
                            with st.container():
                                st.subheader("🛠️ Basic Metrics")
                                performance_data_stats = {
                                    'Metric': [
                                        'Start Balance',
                                        'End Balance',
                                        'Annualized Return (CAGR)',
                                        'Standard Deviation (Annualized)',
                                        'Best Year',
                                        'Worst Year',
                                        'Maximum Drawdown',
                                        'Sharpe Ratio',
                                        'Sortino Ratio',
                                        'Benchmark Correlation'
                                    ],
                                    f'{portfolio_a["name"]}': [
                                        metrics_a['Start Balance'],
                                        metrics_a['End Balance'],
                                        metrics_a['Annualized Return (CAGR)'],
                                        metrics_a['Standard Deviation (Annualized)'],
                                        metrics_a['Best Year'],
                                        metrics_a['Worst Year'],
                                        metrics_a['Maximum Drawdown'],
                                        metrics_a['Sharpe Ratio'],
                                        metrics_a['Sortino Ratio'],
                                        metrics_a['Benchmark Correlation']
                                    ],
                                    f'{portfolio_b["name"]}': [
                                        metrics_b['Start Balance'],
                                        metrics_b['End Balance'],
                                        metrics_b['Annualized Return (CAGR)'],
                                        metrics_b['Standard Deviation (Annualized)'],
                                        metrics_b['Best Year'],
                                        metrics_b['Worst Year'],
                                        metrics_b['Maximum Drawdown'],
                                        metrics_b['Sharpe Ratio'],
                                        metrics_b['Sortino Ratio'],
                                        metrics_b['Benchmark Correlation']
                                    ]
                                }
                                performance_df_stats = pd.DataFrame(performance_data_stats).set_index('Metric')
                                st.table(performance_df_stats)

                        with tabs[2]:
                            st.header("📋 Detail Comparisons")
                            with st.container():
                                st.subheader("🧮 Advanced Metrics")
                                performance_data_advanced = {
                                    'Metric': list(metrics_a.keys()),
                                    f'{portfolio_a["name"]}': list(metrics_a.values()),
                                    f'{portfolio_b["name"]}': list(metrics_b.values())
                                }
                                performance_df_advanced = pd.DataFrame(performance_data_advanced).set_index('Metric')

                                # Replace NaN with empty strings for better visualization
                                performance_df_advanced = performance_df_advanced.replace(np.nan, "")
                                # Drop rows where both Portfolio A and Portfolio B values are NaN
                                performance_df_advanced_clean = performance_df_advanced.dropna(how='all', subset=[f'{portfolio_a["name"]}', f'{portfolio_b["name"]}'])

                                st.table(performance_df_advanced_clean)
                                st.markdown("---")
                                st.write("**Note:** Some metrics may be specific to certain portfolios or may not be directly comparable. Ensure to interpret the metrics in the context of each portfolio's strategy and objectives.")
                            st.subheader("🔍 Risk Factor Attribution Analysis")
                            
                            # Define date range based on the overlapping index
                            start_date = common_dates.min().strftime('%Y-%m-%d')
                            end_date = common_dates.max().strftime('%Y-%m-%d')
                            # Fetch actual factor data
                            factors = fetch_fama_french_factors(start_date, end_date)
                            # Ensure factors are fetched
                            if factors.empty:
                                st.write("Failed to retrieve factor data.")       
                            else:
                                # Align factors frequency with portfolios
                                if isinstance(returns_a.index, pd.DatetimeIndex):
                                    if returns_a.index.freq is None:
                                        inferred_freq = pd.infer_freq(returns_a.index)
                                        if inferred_freq is None:
                                            inferred_freq = 'D'  # Set default frequency to daily
                                    else:
                                        inferred_freq = returns_a.index.freq
                                    
                                    try:
                                        factors = factors.asfreq(inferred_freq, method='ffill')
                                    except TypeError as e:
                                        st.error(f"Error setting frequency for factors: {e}")
                                        st.stop()
                                else:
                                    st.error("Returns index is not a DatetimeIndex.")
                                    st.stop()
                                
                                # Calculate Attribution for Portfolio A
                                attribution_a, error_a = calculate_risk_factor_attribution(returns_a, factors)
                                
                                # Calculate Attribution for Portfolio B
                                attribution_b, error_b = calculate_risk_factor_attribution(returns_b, factors)
                                
                                if error_a or error_b:
                                    st.error("Risk factor attribution analysis is unavailable for one or both portfolios.")
                                else:
                                    if attribution_a.empty or attribution_b.empty:
                                        st.write("Risk factor attribution analysis is unavailable for one or both portfolios.")
                                    else:
                                        st.write(f"**Risk Factor Attribution for {portfolio_a['name']}:**")
                                        st.table(attribution_a)
                                        fig_attribution_a = px.bar(
                                            attribution_a,
                                            x='Factor',
                                            y='Contribution (%)',
                                            title=f'Risk Factor Attribution for {portfolio_a["name"]}',
                                            labels={'Contribution (%)': 'Contribution (%)'},
                                            template='plotly_dark'
                                        )
                                        st.plotly_chart(fig_attribution_a, use_container_width=True)

                                        st.write(f"**Risk Factor Attribution for {portfolio_b['name']}:**")
                                        st.table(attribution_b)
                                        fig_attribution_b = px.bar(
                                            attribution_b,
                                            x='Factor',
                                            y='Contribution (%)',
                                            title=f'Risk Factor Attribution for {portfolio_b["name"]}',
                                            labels={'Contribution (%)': 'Contribution (%)'},
                                            template='plotly_dark'
                                        )
                                        st.plotly_chart(fig_attribution_b, use_container_width=True)

                                        # Combined Bar Chart for Comparison
                                        combined_attribution = pd.concat([attribution_a.set_index('Factor')['Contribution (%)'],
                                                                        attribution_b.set_index('Factor')['Contribution (%)']], axis=1)
                                        combined_attribution.columns = [portfolio_a['name'], portfolio_b['name']]
                                        combined_attribution = combined_attribution.fillna(0)

                                        st.markdown(f"### 📊 Risk Factor Attribution Comparison")
                                        fig_combined_attribution = px.bar(
                                            combined_attribution,
                                            x=combined_attribution.index,
                                            y=combined_attribution.columns,
                                            title='Risk Factor Attribution Comparison',
                                            labels={'value': 'Contribution (%)', 'Factor': 'Risk Factor'},
                                            template='plotly_dark',
                                            barmode='group'
                                        )
                                        st.plotly_chart(fig_combined_attribution, use_container_width=True)

                elif comparison_type == "Portfolio vs Portfolio" and portfolio1 and portfolio2 and portfolio1 != portfolio2:
                    # Handle Portfolio vs Portfolio
                    try:
                        portfolio_a = next(p for p in st.session_state.portfolios if p['name'] == portfolio1)
                        portfolio_b = next(p for p in st.session_state.portfolios if p['name'] == portfolio2)
                    except StopIteration:
                        st.error("One or both selected portfolios were not found.")
                        return

                    # Backtest Portfolio A
                    price_data_a = download_data(
                        portfolio_a['selected'],
                        portfolio_a['start_date'],
                        portfolio_a['end_date']
                    )
                    # Backtest Portfolio B
                    price_data_b = download_data(
                        portfolio_b['selected'],
                        portfolio_b['start_date'],
                        portfolio_b['end_date']
                    )

                    if price_data_a.empty or price_data_b.empty:
                        st.error("No price data available for one or both selected portfolios.")
                    else:
                        # Process Portfolio A
                        available_a = [ticker for ticker in portfolio_a['selected'] if ticker in price_data_a.columns]
                        missing_a = list(set(portfolio_a['selected']) - set(available_a))
                        if missing_a:
                            st.warning(f"Excluded tickers from {portfolio_a['name']} with no data: {', '.join(missing_a)}")
                        if not available_a:
                            st.error(f"No selected tickers have available data for the portfolio '{portfolio_a['name']}'.")
                            return
                        allocations_a = [portfolio_a.get('allocations', [0.0]*len(portfolio_a.get('selected', [])))[i]
                                         for i, ticker in enumerate(portfolio_a.get('selected', [])) if ticker in available_a]
                        if not np.isclose(sum(allocations_a), 100.0, atol=1e-4):
                            st.warning(f"Allocations for {portfolio_a['name']} do not sum to 100%. Adjusting allocations proportionally.")
                            allocations_a = [a / sum(allocations_a) * 100 for a in allocations_a]
                        weights_a = np.array(allocations_a) / 100
                        returns_a, cum_returns_a = backtest(weights_a, price_data_a[available_a],
                                                           portfolio_a['rebalance_freq'], portfolio_a['broker_fee'], debug=False)
                        
                        # Process Portfolio B
                        available_b = [ticker for ticker in portfolio_b['selected'] if ticker in price_data_b.columns]
                        missing_b = list(set(portfolio_b['selected']) - set(available_b))
                        if missing_b:
                            st.warning(f"Excluded tickers from {portfolio_b['name']} with no data: {', '.join(missing_b)}")
                        if not available_b:
                            st.error(f"No selected tickers have available data for the portfolio '{portfolio_b['name']}'.")
                            return
                        allocations_b = [portfolio_b.get('allocations', [0.0]*len(portfolio_b.get('selected', [])))[i]
                                         for i, ticker in enumerate(portfolio_b.get('selected', [])) if ticker in available_b]
                        if not np.isclose(sum(allocations_b), 100.0, atol=1e-4):
                            st.warning(f"Allocations for {portfolio_b['name']} do not sum to 100%. Adjusting allocations proportionally.")
                            allocations_b = [a / sum(allocations_b) * 100 for a in allocations_b]
                        weights_b = np.array(allocations_b) / 100
                        returns_b, cum_returns_b = backtest(weights_b, price_data_b[available_b],
                                                           portfolio_b['rebalance_freq'], portfolio_b['broker_fee'], debug=False)
                        
                        # **Fix: Ensure returns have a DateTime index with matching length**
                        try:
                            returns_a.index = pd.to_datetime(price_data_a.index[:len(returns_a)])
                            cum_returns_a.index = pd.to_datetime(price_data_a.index[:len(cum_returns_a)])
                            returns_b.index = pd.to_datetime(price_data_b.index[:len(returns_b)])
                            cum_returns_b.index = pd.to_datetime(price_data_b.index[:len(cum_returns_b)])
                        except Exception as e:
                            st.error(f"Error converting returns index to DateTimeIndex: {e}")
                            return
                        
                        if cum_returns_a.empty or cum_returns_b.empty:
                            st.error("Cumulative returns are empty for one or both portfolios. Check the data and allocations.")
                            return
                        
                        # Align the data for comparison
                        common_dates = cum_returns_a.index.intersection(cum_returns_b.index)
                        if common_dates.empty:
                            st.error("No overlapping dates between the two portfolios after alignment.")
                            return
                        cum_returns_a = cum_returns_a.loc[common_dates]
                        cum_returns_b = cum_returns_b.loc[common_dates]
                        returns_a = returns_a.loc[common_dates]
                        returns_b = returns_b.loc[common_dates]
                        
                        # Calculate Metrics
                        metrics_a = calculate_portfolio_metrics(
                            returns_a, cum_returns_a, portfolio_a['rf_rate'], None
                        )
                        metrics_b = calculate_portfolio_metrics(
                            returns_b, cum_returns_b, portfolio_b['rf_rate'], None
                        )
                        
                        # Dashboard Header with Key Stats
                        with st.container():
                            st.markdown("### 🔑 Key Metrics")
                            key_metrics = {
                                'Annualized Return (CAGR)': ('CAGR', metrics_a.get('Annualized Return (CAGR)', "N/A"), metrics_b.get('Annualized Return (CAGR)', "N/A")),
                                'Sharpe Ratio': ('Sharpe Ratio', metrics_a.get('Sharpe Ratio', "N/A"), metrics_b.get('Sharpe Ratio', "N/A")),
                                'Maximum Drawdown': ('Maximum Drawdown', metrics_a.get('Maximum Drawdown', "N/A"), metrics_b.get('Maximum Drawdown', "N/A"))
                            }
                            columns = st.columns(len(key_metrics))
                            for (metric, (label, value_a, value_b)) in key_metrics.items():
                                with columns[list(key_metrics.keys()).index(metric)]:
                                    st.metric(label=f"{label} ({portfolio_a['name']})", value=value_a)
                                    st.metric(label=f"{label} ({portfolio_b['name']})", value=value_b)

                        # Create Tabs for Organized Sections
                        tabs = st.tabs(["Overview", "Performance Statistics", "Advanced Metrics", "Drawdowns", "Visualizations"])

                        with tabs[0]:
                            st.header("📈 Portfolios Performance Overview")
                            # Determine the aligned start and end dates
                            aligned_start_date = common_dates.min().strftime('%Y-%m-%d')
                            aligned_end_date = common_dates.max().strftime('%Y-%m-%d')
                            
                            st.write(f"**Aligned Date Range:** {aligned_start_date} to {aligned_end_date}")
                            st.write(f"**Portfolio A:** {portfolio_a['name']}")
                            st.write(f"**Original Start Date:** {portfolio_a['start_date'].strftime('%Y-%m-%d')}")
                            st.write(f"**Portfolio B:** {portfolio_b['name']}")
                            st.write(f"**Original Start Date:** {portfolio_b['start_date'].strftime('%Y-%m-%d')}")

                        with tabs[1]:
                            st.header("📊 Performance Statistics")
                            
                            with st.container():
                                st.subheader("🛠️ Basic Metrics")
                                performance_data_stats = {
                                    'Metric': [
                                        'Start Balance',
                                        'End Balance',
                                        'Annualized Return (CAGR)',
                                        'Standard Deviation (Annualized)',
                                        'Best Year',
                                        'Worst Year',
                                        'Maximum Drawdown',
                                        'Sharpe Ratio',
                                        'Sortino Ratio',
                                        'Benchmark Correlation'
                                    ],
                                    f'{portfolio_a["name"]}': [
                                        metrics_a['Start Balance'],
                                        metrics_a['End Balance'],
                                        metrics_a['Annualized Return (CAGR)'],
                                        metrics_a['Standard Deviation (Annualized)'],
                                        metrics_a['Best Year'],
                                        metrics_a['Worst Year'],
                                        metrics_a['Maximum Drawdown'],
                                        metrics_a['Sharpe Ratio'],
                                        metrics_a['Sortino Ratio'],
                                        metrics_a['Benchmark Correlation']
                                    ],
                                    f'{portfolio_b["name"]}': [
                                        metrics_b['Start Balance'],
                                        metrics_b['End Balance'],
                                        metrics_b['Annualized Return (CAGR)'],
                                        metrics_b['Standard Deviation (Annualized)'],
                                        metrics_b['Best Year'],
                                        metrics_b['Worst Year'],
                                        metrics_b['Maximum Drawdown'],
                                        metrics_b['Sharpe Ratio'],
                                        metrics_b['Sortino Ratio'],
                                        metrics_b['Benchmark Correlation']
                                    ]
                                }
                                performance_df_stats = pd.DataFrame(performance_data_stats).set_index('Metric')
                                st.table(performance_df_stats)

                        with tabs[2]:
                            st.header("📋 Detail Comparisons")
                            with st.container():
                                st.subheader("🧮 Advanced Metrics")
                                performance_data_advanced = {
                                    'Metric': list(metrics_a.keys()),
                                    f'{portfolio_a["name"]}': list(metrics_a.values()),
                                    f'{portfolio_b["name"]}': list(metrics_b.values())
                                }
                                performance_df_advanced = pd.DataFrame(performance_data_advanced).set_index('Metric')

                                # Replace NaN with empty strings for better visualization
                                performance_df_advanced = performance_df_advanced.replace(np.nan, "")
                                # Drop rows where both Portfolio A and Portfolio B values are NaN
                                performance_df_advanced_clean = performance_df_advanced.dropna(how='all', subset=[f'{portfolio_a["name"]}', f'{portfolio_b["name"]}'])

                                st.table(performance_df_advanced_clean)
                                st.markdown("---")
                                st.write("**Note:** Some metrics may be specific to certain portfolios or may not be directly comparable. Ensure to interpret the metrics in the context of each portfolio's strategy and objectives.")
                            st.subheader("🔍 Risk Factor Attribution Analysis")
                            
                            # Define date range based on the overlapping index
                            start_date = common_dates.min().strftime('%Y-%m-%d')
                            end_date = common_dates.max().strftime('%Y-%m-%d')
                            # Fetch actual factor data
                            factors = fetch_fama_french_factors(start_date, end_date)
                            # Ensure factors are fetched
                            if factors.empty:
                                st.write("Failed to retrieve factor data.")       
                            else:
                                # Align factors frequency with portfolios
                                if isinstance(returns_a.index, pd.DatetimeIndex):
                                    if returns_a.index.freq is None:
                                        inferred_freq = pd.infer_freq(returns_a.index)
                                        if inferred_freq is None:
                                            inferred_freq = 'D'  # Set default frequency to daily
                                    else:
                                        inferred_freq = returns_a.index.freq
                                    
                                    try:
                                        factors = factors.asfreq(inferred_freq, method='ffill')
                                    except TypeError as e:
                                        st.error(f"Error setting frequency for factors: {e}")
                                        st.stop()
                                else:
                                    st.error("Returns index is not a DatetimeIndex.")
                                    st.stop()
                                
                                # Calculate Attribution for Portfolio A
                                attribution_a, error_a = calculate_risk_factor_attribution(returns_a, factors)
                                
                                # Calculate Attribution for Portfolio B
                                attribution_b, error_b = calculate_risk_factor_attribution(returns_b, factors)
                                
                                if error_a or error_b:
                                    st.error("Risk factor attribution analysis is unavailable for one or both portfolios.")
                                else:
                                    if attribution_a.empty or attribution_b.empty:
                                        st.write("Risk factor attribution analysis is unavailable for one or both portfolios.")
                                    else:
                                        st.write(f"**Risk Factor Attribution for {portfolio_a['name']}:**")
                                        st.table(attribution_a)
                                        fig_attribution_a = px.bar(
                                            attribution_a,
                                            x='Factor',
                                            y='Contribution (%)',
                                            title=f'Risk Factor Attribution for {portfolio_a["name"]}',
                                            labels={'Contribution (%)': 'Contribution (%)'},
                                            template='plotly_dark'
                                        )
                                        st.plotly_chart(fig_attribution_a, use_container_width=True)

                                        st.write(f"**Risk Factor Attribution for {portfolio_b['name']}:**")
                                        st.table(attribution_b)
                                        fig_attribution_b = px.bar(
                                            attribution_b,
                                            x='Factor',
                                            y='Contribution (%)',
                                            title=f'Risk Factor Attribution for {portfolio_b["name"]}',
                                            labels={'Contribution (%)': 'Contribution (%)'},
                                            template='plotly_dark'
                                        )
                                        st.plotly_chart(fig_attribution_b, use_container_width=True)

                                        # Combined Bar Chart for Comparison
                                        combined_attribution = pd.concat([attribution_a.set_index('Factor')['Contribution (%)'],
                                                                        attribution_b.set_index('Factor')['Contribution (%)']], axis=1)
                                        combined_attribution.columns = [portfolio_a['name'], portfolio_b['name']]
                                        combined_attribution = combined_attribution.fillna(0)

                                        st.markdown(f"### 📊 Risk Factor Attribution Comparison")
                                        fig_combined_attribution = px.bar(
                                            combined_attribution,
                                            x=combined_attribution.index,
                                            y=combined_attribution.columns,
                                            title='Risk Factor Attribution Comparison',
                                            labels={'value': 'Contribution (%)', 'Factor': 'Risk Factor'},
                                            template='plotly_dark',
                                            barmode='group'
                                        )
                                        st.plotly_chart(fig_combined_attribution, use_container_width=True)

                st.success("Backtest completed successfully!")
    else:
        st.warning("Please add at least one portfolio to run backtest.")

# metrics.py
import numpy as np
import pandas as pd
import statsmodels.api as sm
import streamlit as st
from scipy.stats import skew, kurtosis

def calculate_sharpe_ratio(returns, rf=0.02):
    """
    Calculate the Sharpe Ratio for a given set of returns.
    """
    excess_return = returns.mean() * 252 - rf
    std_dev = returns.std() * np.sqrt(252)
    return excess_return / std_dev if std_dev != 0 else np.nan

def calculate_sortino_ratio(returns, rf=0.02):
    """
    Calculate the Sortino Ratio for a given set of returns.
    """
    excess_return = returns.mean() * 252 - rf
    downside_std = returns[returns < 0].std() * np.sqrt(252)
    return excess_return / downside_std if downside_std != 0 else np.nan

def calculate_beta(returns, benchmark_returns):
    """
    Calculate the Beta of the portfolio relative to the benchmark.
    """
    if returns.empty or benchmark_returns.empty:
        return np.nan
    covariance_matrix = np.cov(returns, benchmark_returns)
    covariance = covariance_matrix[0, 1]
    benchmark_variance = covariance_matrix[1, 1]
    return covariance / benchmark_variance if benchmark_variance != 0 else np.nan

def calculate_alpha(returns, benchmark_returns, rf=0.02):
    """
    Calculate the Alpha of the portfolio relative to the benchmark.
    """
    beta = calculate_beta(returns, benchmark_returns)
    portfolio_return = returns.mean() * 252
    benchmark_return = benchmark_returns.mean() * 252
    return portfolio_return - (rf + beta * (benchmark_return - rf)) if not np.isnan(beta) else np.nan

def calculate_treynor_ratio(returns, benchmark_returns, rf=0.02):
    """
    Calculate the Treynor Ratio for a given set of returns.
    """
    beta = calculate_beta(returns, benchmark_returns)
    excess_return = returns.mean() * 252 - rf
    return excess_return / beta if beta != 0 else np.nan

def calculate_calmar_ratio(returns, cum_returns):
    """
    Calculate the Calmar Ratio for a given set of returns.
    """
    annual_return = returns.mean() * 252
    max_dd = drawdown(cum_returns)
    return annual_return / abs(max_dd) if max_dd != 0 else np.nan

def calculate_r_squared(returns, benchmark_returns):
    """
    Calculate the R-squared of the portfolio relative to the benchmark.
    """
    if returns.empty or benchmark_returns.empty:
        return np.nan
    covariance = np.cov(returns, benchmark_returns)
    var_port = covariance[0,0]
    var_bench = covariance[1,1]
    cov = covariance[0,1]
    return (cov ** 2) / (var_port * var_bench) if var_port !=0 and var_bench !=0 else np.nan

def calculate_information_ratio(returns, benchmark_returns):
    """
    Calculate the Information Ratio of the portfolio relative to the benchmark.
    """
    if returns.empty or benchmark_returns.empty:
        return np.nan
    active_return = (returns.mean() - benchmark_returns.mean()) * 252
    tracking_error = calculate_tracking_error(returns, benchmark_returns)
    return active_return / tracking_error if tracking_error != 0 else np.nan

def calculate_tracking_error(returns, benchmark_returns):
    """
    Calculate the Tracking Error of the portfolio relative to the benchmark.
    """
    if returns.empty or benchmark_returns.empty:
        return np.nan
    return np.std((returns - benchmark_returns)) * np.sqrt(252)

def calculate_active_return(returns, benchmark_returns):
    """
    Calculate the Active Return of the portfolio relative to the benchmark.
    """
    if returns.empty or benchmark_returns.empty:
        return np.nan
    return (returns.mean() - benchmark_returns.mean()) * 252 * 100

def calculate_gain_loss_ratio(returns):
    """
    Calculate the Gain/Loss Ratio of the portfolio.
    """
    gains = returns[returns > 0].sum()
    losses = -returns[returns < 0].sum()
    return gains / losses if losses != 0 else np.nan

def drawdown(cum_returns):
    """
    Calculate the Maximum Drawdown of the portfolio.
    """
    if cum_returns.empty:
        return np.nan
    peak = cum_returns.expanding(min_periods=1).max()
    dd = (cum_returns / peak) - 1
    return dd.min()

def calculate_capture_ratio(returns, benchmark_returns, upside=True):
    """
    Calculate the Upside or Downside Capture Ratio of the portfolio.
    """
    if returns.empty or benchmark_returns.empty:
        return np.nan
    mask = benchmark_returns > 0 if upside else benchmark_returns < 0
    if mask.sum() == 0:
        return np.nan
    portfolio = returns[mask]
    benchmark = benchmark_returns[mask]
    return (portfolio.sum() / benchmark.sum()) * 100 if benchmark.sum() != 0 else np.nan

def calculate_safe_withdrawal_rate(returns):
    """
    Calculate the Safe Withdrawal Rate based on portfolio returns.
    """
    if returns.empty:
        return np.nan
    return (returns.mean() / returns.std()) * 100

def calculate_perpetual_withdrawal_rate(returns):
    """
    Calculate the Perpetual Withdrawal Rate based on portfolio returns.
    """
    if returns.empty:
        return np.nan
    return (returns.mean() / returns.std()) * 100

def calculate_positive_periods(returns):
    """
    Calculate the number and percentage of positive return periods.
    """
    if returns.empty:
        return "N/A"
    positive = returns > 0
    return f"{positive.sum()} out of {len(returns)} ({(positive.sum()/len(returns))*100:.2f}%)"

def calculate_modigliani_miller(returns, benchmark_returns, rf=0.02):
    """
    Calculate the Modigliani–Modigliani Measure (M²) of the portfolio.
    """
    sharpe = calculate_sharpe_ratio(returns, rf)
    alpha = calculate_alpha(returns, benchmark_returns, rf)
    return alpha / sharpe if sharpe != 0 and not np.isnan(alpha) else np.nan

def calculate_beta_local(portfolio_returns, benchmark_returns):
    """
    Local version of calculate_beta to avoid naming conflicts.
    """
    if portfolio_returns.empty or benchmark_returns.empty:
        return np.nan
    covariance_matrix = np.cov(portfolio_returns, benchmark_returns)
    covariance = covariance_matrix[0, 1]
    benchmark_variance = covariance_matrix[1, 1]
    return covariance / benchmark_variance if benchmark_variance != 0 else np.nan

def calculate_var(returns, confidence_level=0.95):
    """
    Calculate the Value at Risk (VaR) for a given set of returns.
    """
    return np.percentile(returns, 100 * (1 - confidence_level))

def calculate_cvar(returns, confidence_level=0.95):
    """
    Calculate the Conditional Value at Risk (CVaR) for a given set of returns.
    """
    var = calculate_var(returns, confidence_level)
    return returns[returns <= var].mean()

def calculate_simple_metrics(returns, cum_returns, start_balance=10000):
    metrics = {}
    metrics['Start Balance'] = f"${start_balance:,.2f}"
    if not cum_returns.empty:
        try:
            end_balance = start_balance * cum_returns.iloc[-1]
            metrics['End Balance'] = f"${end_balance:,.2f}"
            start_date = cum_returns.index[0]
            end_date = cum_returns.index[-1]
            days = (end_date - start_date).days
            years = days / 365.25
            cagr = (end_balance / start_balance) ** (1 / years) - 1 if years > 0 else np.nan
            metrics['Annualized Return (CAGR)'] = f"{cagr * 100:.2f}%" if not np.isnan(cagr) else "N/A"
            yearly_returns = returns.resample('Y').apply(lambda x: (1 + x).prod() - 1)
            best_year = yearly_returns.max() * 100 if not yearly_returns.empty else np.nan
            worst_year = yearly_returns.min() * 100 if not yearly_returns.empty else np.nan
            metrics['Best Year'] = f"{best_year:.2f}%" if not np.isnan(best_year) else "N/A"
            metrics['Worst Year'] = f"{worst_year:.2f}%" if not np.isnan(worst_year) else "N/A"
            metrics['Arithmetic Mean (Monthly)'] = f"{returns.mean() * 100:.2f}%" if not returns.empty else np.nan
            metrics['Arithmetic Mean (Annualized)'] = f"{returns.mean() * 252 * 100:.2f}%" if not returns.empty else np.nan
            metrics['Geometric Mean (Monthly)'] = f"{(np.exp(np.log1p(returns).mean()) - 1) * 100:.2f}%" if not returns.empty else np.nan
            metrics['Geometric Mean (Annualized)'] = f"{(np.exp(np.log1p(returns).mean() * 252) - 1) * 100:.2f}%" if not returns.empty else np.nan
            metrics['Standard Deviation (Monthly)'] = f"{returns.std() * 100:.2f}%" if not returns.empty else np.nan
            metrics['Standard Deviation (Annualized)'] = f"{returns.std() * np.sqrt(252) * 100:.2f}%" if not returns.empty else np.nan
            metrics['Downside Deviation (Monthly)'] = f"{returns[returns < 0].std() * 100:.2f}%" if not returns.empty else np.nan
            metrics['Maximum Drawdown'] = f"{drawdown(cum_returns) * 100:.2f}%" if not cum_returns.empty else np.nan
            metrics['Sharpe Ratio'] = f"{calculate_sharpe_ratio(returns, 0.02):.2f}" if not cum_returns.empty else np.nan
            metrics['Sortino Ratio'] = f"{calculate_sortino_ratio(returns, 0.02):.2f}" if not cum_returns.empty else np.nan
            metrics['Gain/Loss Ratio'] = f"{calculate_gain_loss_ratio(returns):.2f}" if not cum_returns.empty else np.nan
            metrics['Skewness'] = f"{skew(returns):.2f}" if not returns.empty else np.nan
            metrics['Excess Kurtosis'] = f"{kurtosis(returns):.2f}" if not returns.empty else np.nan
            metrics['Safe Withdrawal Rate'] = f"{calculate_safe_withdrawal_rate(returns):.6f}%" if not returns.empty else np.nan
            metrics['Perpetual Withdrawal Rate'] = f"{calculate_perpetual_withdrawal_rate(returns):.6f}%" if not returns.empty else np.nan
            metrics['Positive Periods'] = calculate_positive_periods(returns) if not returns.empty else "N/A"
        except Exception as e:
            st.error(f"Error calculating simple metrics: {e}")
            for key in ['End Balance', 'Annualized Return (CAGR)', 'Best Year', 'Worst Year',
                        'Arithmetic Mean (Monthly)', 'Arithmetic Mean (Annualized)',
                        'Geometric Mean (Monthly)', 'Geometric Mean (Annualized)',
                        'Standard Deviation (Monthly)', 'Standard Deviation (Annualized)',
                        'Downside Deviation (Monthly)', 'Maximum Drawdown',
                        'Sharpe Ratio', 'Sortino Ratio', 'Gain/Loss Ratio',
                        'Skewness', 'Excess Kurtosis', 'Safe Withdrawal Rate',
                        'Perpetual Withdrawal Rate', 'Positive Periods']:
                metrics[key] = np.nan  # Use NaN instead of "N/A"
    else:
        for key in ['End Balance', 'Annualized Return (CAGR)', 'Best Year', 'Worst Year',
                    'Arithmetic Mean (Monthly)', 'Arithmetic Mean (Annualized)',
                    'Geometric Mean (Monthly)', 'Geometric Mean (Annualized)',
                    'Standard Deviation (Monthly)', 'Standard Deviation (Annualized)',
                    'Downside Deviation (Monthly)', 'Maximum Drawdown',
                    'Sharpe Ratio', 'Sortino Ratio', 'Gain/Loss Ratio',
                    'Skewness', 'Excess Kurtosis', 'Safe Withdrawal Rate',
                    'Perpetual Withdrawal Rate', 'Positive Periods']:
            metrics[key] = np.nan  # Use NaN instead of "N/A"
    
    return metrics

@st.cache_data(show_spinner=False)
def calculate_portfolio_metrics(returns, cum_returns, rf, benchmark_returns=None):
    metrics = calculate_simple_metrics(returns, cum_returns)
    
    if benchmark_returns is not None and not benchmark_returns.empty:
        correlation = returns.corr(benchmark_returns)
        beta = calculate_beta(returns, benchmark_returns)
        metrics['Benchmark Correlation'] = f"{correlation:.2f}" if not np.isnan(correlation) else np.nan
        metrics['Beta'] = f"{beta:.2f}" if not np.isnan(beta) else np.nan

        alpha = calculate_alpha(returns, benchmark_returns, rf)
        metrics['Alpha (annualized)'] = f"{alpha * 100:.2f}%" if not np.isnan(alpha) else np.nan

        r2 = calculate_r_squared(returns, benchmark_returns)
        metrics['R2'] = f"{r2 * 100:.2f}%" if not np.isnan(r2) else np.nan

        treynor = calculate_treynor_ratio(returns, benchmark_returns, rf)
        metrics['Treynor Ratio'] = f"{treynor:.2f}" if not np.isnan(treynor) else np.nan

        calmar = calculate_calmar_ratio(returns, cum_returns)
        metrics['Calmar Ratio'] = f"{calmar:.2f}" if not np.isnan(calmar) else np.nan

        m2 = calculate_modigliani_miller(returns, benchmark_returns, rf)
        metrics['Modigliani–Modigliani Measure'] = f"{m2 * 100:.2f}%" if not np.isnan(m2) else np.nan

        info_ratio = calculate_information_ratio(returns, benchmark_returns)
        metrics['Information Ratio'] = f"{info_ratio:.2f}" if not np.isnan(info_ratio) else np.nan

        tracking_error = calculate_tracking_error(returns, benchmark_returns)
        metrics['Tracking Error'] = f"{tracking_error * 100:.2f}%" if not np.isnan(tracking_error) else np.nan

        active_return = calculate_active_return(returns, benchmark_returns)
        metrics['Active Return'] = f"{active_return:.2f}%" if not np.isnan(active_return) else np.nan

        upside_capture = calculate_capture_ratio(returns, benchmark_returns, upside=True)
        downside_capture = calculate_capture_ratio(returns, benchmark_returns, upside=False)
        metrics['Upside Capture Ratio'] = f"{upside_capture:.2f}%" if not np.isnan(upside_capture) else np.nan
        metrics['Downside Capture Ratio'] = f"{downside_capture:.2f}%" if not np.isnan(downside_capture) else np.nan
    else:
        metrics.update({
            'Benchmark Correlation': np.nan,
            'Beta': np.nan,
            'Alpha (annualized)': np.nan,
            'R2': np.nan,
            'Treynor Ratio': np.nan,
            'Calmar Ratio': np.nan,
            'Modigliani–Modigliani Measure': np.nan,
            'Information Ratio': np.nan,
            'Tracking Error': np.nan,
            'Active Return': np.nan,
            'Upside Capture Ratio': np.nan,
            'Downside Capture Ratio': np.nan
        })
    
    return metrics

def calculate_risk_factor_attribution(returns, factors, annualization_factor=252):
    """
    Decompose portfolio returns by risk factors using linear regression and provide
    comments on the factor contributions.

    Parameters:
    - returns (pd.Series): Portfolio returns.
    - factors (pd.DataFrame): DataFrame where each column represents a risk factor.
    - annualization_factor (int): Annualization factor (252 for daily, 12 for monthly returns).

    Returns:
    - pd.DataFrame: Factor names, their contribution percentages, and explanatory comments.
    - str: Error message if any exception occurs, else None.
    """
    try:
        # Ensure returns has a name, or assign one if missing
        if returns.name is None:
            returns = returns.rename("Portfolio_Returns")
        
        # Convert returns to a DataFrame
        returns = returns.to_frame()
        
        # Ensure both returns and factors are in decimal format
        if returns['Portfolio_Returns'].max() > 1:
            returns = returns / 100
            
        if factors.max().max() > 1:
            factors = factors / 100
        
        # Adjust returns to excess returns by subtracting RF (if it exists in factors)
        if 'RF' in factors.columns:
            returns['Portfolio_Returns'] = returns['Portfolio_Returns'] - factors['RF']
            factors = factors.drop(columns=['RF'])
        
        # Align the data using an inner join and drop any remaining NaN values
        aligned_data = pd.merge(returns, factors, left_index=True, right_index=True, how='inner').dropna()

        # Check that we have data after merging
        if aligned_data.empty:
            raise ValueError("No overlapping dates between returns and factors after dropping NaNs.")
    
        # Prepare X and y for regression
        X = aligned_data[factors.columns]
        X = sm.add_constant(X)  # Add intercept
        y = aligned_data['Portfolio_Returns']
        
        # Run the regression model
        model = sm.OLS(y, X).fit()
        coefficients = model.params.drop('const', errors='ignore')
        
        # Calculate factor contributions (non-annualized)
        factor_means = X.mean().drop('const', errors='ignore')
        contributions = coefficients * factor_means
        
        # Calculate total return
        total_return = returns['Portfolio_Returns'].mean()

        # Check if total_return is zero to avoid division by zero
        if np.isclose(total_return, 0):
            attribution_df = pd.DataFrame({
                'Factor': contributions.index,
                'Contribution (%)': [0.0] * len(contributions)
            })
        else:
            # Calculate percentage contributions
            attribution_df = pd.DataFrame({
                'Factor': contributions.index,
                'Contribution (%)': (contributions / total_return * 100).round(2)
            })
        
        # Remove the constant term if it exists
        if 'const' in attribution_df['Factor'].values:
            attribution_df = attribution_df[attribution_df['Factor'] != 'const']
        
        # Use full names for factors
        factor_full_names = {
            'Mkt-RF': 'Market Risk Premium',
            'SMB': 'Small Minus Big',
            'HML': 'High Minus Low'
        }
        attribution_df['Factor'] = attribution_df['Factor'].replace(factor_full_names)
        
        # Generate detailed comments based on factor contributions
        comments = []
        for _, row in attribution_df.iterrows():
            factor_name = row['Factor']
            contribution = row['Contribution (%)']

            # Market Risk Premium (Mkt-RF) Analysis
            if factor_name == 'Market Risk Premium':
                if contribution > 75:
                    comments.append(f"{factor_name} = {contribution}%: Extremely high sensitivity to market movements.")
                elif contribution > 50:
                    comments.append(f"{factor_name} = {contribution}%: High sensitivity to overall market movements.")
                elif contribution > 30:
                    comments.append(f"{factor_name} = {contribution}%: Notable sensitivity to market risk.")
                elif contribution > 10:
                    comments.append(f"{factor_name} = {contribution}%: Moderate exposure to market risk.")
                else:
                    comments.append(f"{factor_name} = {contribution}%: Low exposure to market risk.")

            # Small Minus Big (SMB) Analysis
            elif factor_name == 'Small Minus Big':
                if contribution > 5:
                    comments.append(f"{factor_name} = {contribution}%: Strong tilt towards small-cap stocks.")
                elif contribution > 0:
                    comments.append(f"{factor_name} = {contribution}%: Mild preference for small-cap stocks.")
                elif contribution > -5:
                    comments.append(f"{factor_name} = {contribution}%: Slight tilt towards larger-cap stocks.")
                else:
                    comments.append(f"{factor_name} = {contribution}%: Clear preference for large-cap stocks.")

            # High Minus Low (HML) Analysis
            elif factor_name == 'High Minus Low':
                if contribution > 5:
                    comments.append(f"{factor_name} = {contribution}%: Strong tilt toward value stocks.")
                elif contribution > 0:
                    comments.append(f"{factor_name} = {contribution}%: Slight positive tilt toward value stocks.")
                elif contribution > -5:
                    comments.append(f"{factor_name} = {contribution}%: Minor negative tilt toward growth stocks.")
                else:
                    comments.append(f"{factor_name} = {contribution}%: Clear preference for growth stocks.")

        # Add comments to the DataFrame
        attribution_df['Comment'] = comments
    
        return attribution_df, None
    
    except Exception as e:
        error_message = f"Error calculating risk factor attribution: {e}"
        return pd.DataFrame(), error_message

def calculate_performance_attribution(returns, weights):
    """
    Calculate the performance attribution of the portfolio.

    Parameters:
    - returns (pd.DataFrame or pd.Series): Historical returns of assets.
    - weights (list or np.ndarray): Weights of each asset in the portfolio.

    Returns:
    - pd.DataFrame: Asset names, their contribution percentages, and explanatory comments.
    """
    try:
        if isinstance(returns, pd.Series):
            returns = returns.to_frame('Asset')  # Convert Series to DataFrame with a default column name
        
        if len(weights) != len(returns.columns):
            st.error(f"Number of weights ({len(weights)}) does not match number of assets ({len(returns.columns)}).")
            return pd.DataFrame()
        
        annual_returns = returns.mean() * 252
        contributions = annual_returns * weights
        attribution_df = pd.DataFrame({
            'Asset': returns.columns,
            'Contribution (%)': (contributions / contributions.sum() * 100).round(2)
        }).sort_values(by='Contribution (%)', ascending=False)
        return attribution_df
    except Exception as e:
        st.error(f"Error calculating performance attribution: {e}")
        return pd.DataFrame()

def calculate_final_score(primary_metrics, comparison_metrics):
    """
    Calculate the final score based on selected metrics.

    Parameters:
    - primary_metrics (dict): Metrics of the primary portfolio.
    - comparison_metrics (dict): Metrics of the comparison portfolio.

    Returns:
    - float: Final score.
    """
    metrics_to_compare = [
        'Annualized Return (CAGR)',
        'Sharpe Ratio',
        'Sortino Ratio',
        'Treynor Ratio',
        'Calmar Ratio',
        'Alpha (annualized)',
        'Information Ratio',
        'Modigliani–Modigliani Measure',
        'Upside Capture Ratio',
        'Gain/Loss Ratio'
    ]
    metric_weights = {
        'Annualized Return (CAGR)': 15,
        'Sharpe Ratio': 15,
        'Sortino Ratio': 10,
        'Treynor Ratio': 10,
        'Calmar Ratio': 10,
        'Alpha (annualized)': 10,
        'Information Ratio': 10,
        'Modigliani–Modigliani Measure': 10,
        'Upside Capture Ratio': 5,
        'Gain/Loss Ratio': 5
    }

    score = 0
    total = 0

    for metric in metrics_to_compare:
        primary = primary_metrics.get(metric, "N/A")
        comparison = comparison_metrics.get(metric, "N/A")
        if primary != "N/A" and comparison != "N/A":
            primary_val = float(primary.strip('%')) if isinstance(primary, str) and '%' in primary else float(primary)
            comparison_val = float(comparison.strip('%')) if isinstance(comparison, str) and '%' in comparison else float(comparison)
            weight = metric_weights.get(metric, 1)
            if comparison_val == 0:
                continue
            if primary_val > comparison_val:
                score += weight * (primary_val / comparison_val)
            else:
                score += weight * (primary_val / comparison_val) * 0.5  # Partial credit
            total += weight
    return (score / total) * 100 if total > 0 else 0

def get_drawdown_details(cum_returns):
    """
    Calculate detailed drawdown information for the portfolio.

    Parameters:
    - cum_returns (pd.Series): Cumulative returns of the portfolio.

    Returns:
    - list: Detailed drawdown information.
    """
    drawdowns = []
    if cum_returns.empty:
        return drawdowns
    peak = cum_returns.iloc[0]
    peak_date = cum_returns.index[0]
    trough = cum_returns.iloc[0]
    trough_date = cum_returns.index[0]
    for date, value in cum_returns.items():
        if value > peak:
            if trough < peak:
                recovery = cum_returns[cum_returns >= peak].loc[trough_date:]
                if not recovery.empty:
                    recovery_date = recovery.index[0]
                    if recovery_date > trough_date:
                        recovery_time = (recovery_date - trough_date).days
                        underwater_period = (recovery_date - peak_date).days
                        drawdowns.append({
                            'Start': peak_date.strftime('%b %Y'),
                            'End': trough_date.strftime('%b %Y'),
                            'Length': f"{(trough_date - peak_date).days // 30} months",
                            'Recovery By': recovery_date.strftime('%b %Y'),
                            'Recovery Time': f"{recovery_time // 30} months",
                            'Underwater Period': f"{underwater_period // 30} months",
                            'Drawdown': f"{((trough / peak) - 1) * 100:.2f}%"
                        })
                    else:
                        drawdowns.append({
                            'Start': peak_date.strftime('%b %Y'),
                            'End': trough_date.strftime('%b %Y'),
                            'Length': f"{(trough_date - peak_date).days // 30} months",
                            'Recovery By': "Not Recovered",
                            'Recovery Time': "N/A",
                            'Underwater Period': "N/A",
                            'Drawdown': f"{((trough / peak) - 1) * 100:.2f}%"
                        })
            peak = value
            peak_date = date
            trough = value
            trough_date = date
        elif value < trough:
            trough = value
            trough_date = date
    if trough < peak:
        drawdowns.append({
            'Start': peak_date.strftime('%b %Y'),
            'End': trough_date.strftime('%b %Y'),
            'Length': f"{(trough_date - peak_date).days // 30} months",
            'Recovery By': "Not Recovered",
            'Recovery Time': "N/A",
            'Underwater Period': "N/A",
            'Drawdown': f"{((trough / peak) - 1) * 100:.2f}%"
        })
    try:
        drawdowns_sorted = sorted(drawdowns, key=lambda x: float(x['Drawdown'].strip('%')), reverse=False)
    except:
        drawdowns_sorted = []
    drawdowns_sorted = drawdowns_sorted[:10]
    for idx, dd in enumerate(drawdowns_sorted, start=1):
        dd['Rank'] = idx
    drawdowns_final = []
    for dd in drawdowns_sorted:
        drawdowns_final.append({
            'Rank': dd['Rank'],
            'Start': dd['Start'],
            'End': dd['End'],
            'Length': dd['Length'],
            'Recovery By': dd['Recovery By'],
            'Recovery Time': dd['Recovery Time'],
            'Underwater Period': dd['Underwater Period'],
            'Drawdown': dd['Drawdown']
        })
    return drawdowns_final

# utils.py
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from pandas_datareader import data as pdr
import yfinance as yf
import pandas as pd
import streamlit as st

frequency_mapping = {
    "Daily": "D",
    "Weekly": "W",
    "Monthly": "M",
    "Quarterly": "Q",
    "Yearly": "Y"
}

def calculate_cvar(portfolio_returns, confidence_level=0.95):
    var = calculate_var(portfolio_returns, confidence_level)
    portfolio_returns = np.array(portfolio_returns)
    return portfolio_returns[portfolio_returns <= var].mean()
    
def calculate_var(portfolio_returns, confidence_level=0.95):
    if not isinstance(portfolio_returns, np.ndarray):
        portfolio_returns = np.array(portfolio_returns)
    return np.percentile(portfolio_returns, (1 - confidence_level) * 100)

def get_company_name(ticker_df, ticker):
    """
    Retrieve the company name based on the ticker symbol.
    """
    match = ticker_df[ticker_df['Ticker'] == ticker]
    if not match.empty:
        return match.iloc[0]['Company Name']
    return "Unknown"

def format_asset_option(ticker, company_name):
    """
    Format the asset option string for display.
    """
    return f"{ticker} - {company_name}"

@st.cache_data(show_spinner=False)
def get_tickers():
    """
    Fetch and combine S&P 500 and NASDAQ-100 tickers from Wikipedia.
    """
    try:
        # Fetch S&P 500 companies
        sp500_url = 'https://en.wikipedia.org/wiki/List_of_S%26P_500_companies'
        sp500_response = requests.get(sp500_url, verify=False)
        sp500_table = pd.read_html(sp500_response.text)[0]
        sp500 = sp500_table[['Symbol', 'Security']].rename(columns={'Symbol': 'Ticker', 'Security': 'Company Name'})
        sp500['Ticker'] = sp500['Ticker'].str.replace('.', '-', regex=False)

        # Fetch NASDAQ-100 companies
        nasdaq100_url = 'https://en.wikipedia.org/wiki/NASDAQ-100'
        nasdaq100_response = requests.get(nasdaq100_url, verify=False)
        nasdaq100_tables = pd.read_html(nasdaq100_response.text)
        
        nasdaq100 = pd.DataFrame()
        for table in nasdaq100_tables:
            if 'Ticker' in table.columns and 'Company' in table.columns:
                nasdaq100 = table[['Ticker', 'Company']].rename(columns={'Ticker': 'Ticker', 'Company': 'Company Name'})
                nasdaq100['Ticker'] = nasdaq100['Ticker'].str.replace('.', '-', regex=False)
                break

        combined = pd.concat([sp500, nasdaq100], ignore_index=True)
        combined = combined.drop_duplicates(subset=['Ticker'])
        return combined.sort_values('Ticker').reset_index(drop=True)
    except Exception as e:
        st.error(f"Error fetching tickers: {e}")
        return pd.DataFrame(columns=['Ticker', 'Company Name'])

@st.cache_data(show_spinner=False)
def download_data(tickers, start, end, retries=3, backoff_factor=0.3):
    """
    Download historical price data for given tickers using yfinance with retry strategy.
    """
    try:
        # Configure retry strategy for requests
        session = requests.Session()
        retry = Retry(
            total=retries,
            read=retries,
            connect=retries,
            backoff_factor=backoff_factor,
            status_forcelist=(500, 502, 504),
        )
        adapter = HTTPAdapter(max_retries=retry)
        session.mount('http://', adapter)
        session.mount('https://', adapter)
        
        # Attempt to download data
        data = yf.download(tickers, start=start, end=end, progress=False, session=session)['Adj Close']
        
        # Handle potential empty data
        if isinstance(data, pd.Series):
            data = data.to_frame()
        if data.empty:
            st.warning("No price data available for the selected portfolio. Please check the ticker symbols and date range.")
            return pd.DataFrame()
        if data.index.tz is not None:
            data.index = data.index.tz_localize(None)
        
        # Fill missing data
        data = data.fillna(method='ffill').fillna(method='bfill')
        if data.isnull().values.any():
            st.warning("Data contains missing values after filling. Some calculations may be affected.")
        return data
    except Exception as e:
        st.error(f"Error downloading data: {e}")
        return pd.DataFrame()

def get_common_benchmarks(selected_tickers):
    """
    Suggest common benchmarks based on the selected tickers.
    """
    # Define benchmark suggestions based on asset sectors or indices
    sp500 = {"AAPL", "MSFT", "GOOGL", "AMZN", "META", "TSLA"}  # Example S&P 500 tech companies
    nasdaq_tech = {"AAPL", "MSFT", "GOOGL", "AMZN", "META", "TSLA", "NVDA", "AMD"}

    selected_set = set(selected_tickers)

    if selected_set.issubset(sp500):
        return {
            "S&P 500": "^GSPC",
            "Dow Jones Industrial Average": "^DJI",
            "Russell 2000": "^RUT",
            "Custom": "CUSTOM"
        }
    elif selected_set.issubset(nasdaq_tech):
        return {
            "NASDAQ Composite": "^IXIC",
            "QQQ (Invesco QQQ ETF)": "QQQ",
            "Custom": "CUSTOM"
        }
    else:
        return {
            "S&P 500": "^GSPC",
            "NASDAQ Composite": "^IXIC",
            "Dow Jones Industrial Average": "^DJI",
            "Russell 2000": "^RUT",
            "Custom": "CUSTOM"
        }

def generate_recommendations(allocations, available_selected):
    """
    Generate basic portfolio recommendations based on current allocations.
    """
    recommendations = []
    max_alloc = max(allocations)
    min_alloc = min(allocations)
    total_alloc = sum(allocations)
    
    if max_alloc > 40.0:
        idx = allocations.index(max_alloc)
        ticker = available_selected[idx]
        recommendations.append(f"🔹 Consider reducing the allocation to **{ticker}** since it constitutes **{max_alloc:.2f}%** of your portfolio.")
    
    if min_alloc < 5.0:
        idx = allocations.index(min_alloc)
        ticker = available_selected[idx]
        recommendations.append(f"🔸 Consider increasing the allocation to **{ticker}** to at least **5.00%** for better diversification.")
    
    if total_alloc < 100.0:
        recommendations.append("🔹 Consider allocating the remaining funds to additional assets to reach a total of **100%**.")
    
    return recommendations

def fetch_fama_french_factors(start_date, end_date):
    """
    Fetch daily Fama-French factors from a data source.
    """
    try:
        # Fetch daily Fama-French factors
        ff_data = pdr.get_data_famafrench('F-F_Research_Data_Factors_daily', start=start_date, end=end_date)
        if not ff_data:
            st.error("Fama-French data not found.")
            return pd.DataFrame()
        
        factors = ff_data[0]
        factors = factors.rename(columns=lambda x: x.strip())  # Remove any leading/trailing spaces
        factors /= 100  # Convert percentage returns to decimal format
        
        # Fill missing data by forward and backward filling
        factors = factors.fillna(method='ffill').fillna(method='bfill')
        
        # Check for necessary columns
        required_columns = {'Mkt-RF', 'SMB', 'HML', 'RF'}
        if not required_columns.issubset(factors.columns):
            missing = required_columns - set(factors.columns)
            st.error(f"Missing factor columns: {', '.join(missing)}")
            return pd.DataFrame()
        
        return factors
    except Exception as e:
        st.error(f"Error fetching factor data: {e}")
        return pd.DataFrame()
        
def generate_portfolio(returns, rf):
    try:
        weights = np.random.random(returns.shape[1])
        weights /= np.sum(weights)
        portfolio_return = np.dot(weights, returns.mean()) * 252
        portfolio_volatility = np.sqrt(np.dot(weights.T, np.dot(returns.cov() * 252, weights)))
        sharpe_ratio = (portfolio_return - rf) / portfolio_volatility if portfolio_volatility > 0 else 0
        return portfolio_volatility, portfolio_return, sharpe_ratio
    except Exception as e:
        st.error(f"Error generating portfolio: {e}")
        return np.nan, np.nan, np.nan  # Return NaNs to indicate failure
        
def monte_carlo_simulation(returns, num_simulations=1000, periods=252, mean_returns=None, cov_matrix=None,
                           mean_reversion=False, mean_reversion_speed=0.1, long_term_mean=None,
                           time_varying_vol=False, vol_change_rate=0.0,
                           stress_shocks=None, stress_period=None,
                           return_distribution='normal'):
    if mean_returns is None:
        mean_returns = returns.mean()
    if cov_matrix is None:
        cov_matrix = returns.cov()

    num_assets = returns.shape[1]
    available_selected = returns.columns.tolist()

    def simulate():
        weights = np.random.random(num_assets)
        weights /= np.sum(weights)
        simulated_prices = [1]  # Start with initial price of 1
        current_mean = mean_returns.copy()
        current_vol = np.sqrt(np.diag(cov_matrix))
        for t in range(periods):
            if mean_reversion:
                current_mean += mean_reversion_speed * (long_term_mean - current_mean)
            if time_varying_vol:
                current_vol += vol_change_rate
            adjusted_cov_matrix = np.outer(current_vol, current_vol) * np.corrcoef(returns.T)
            if return_distribution == 'log-normal':
                simulated_returns = np.random.lognormal(mean=np.log(1 + current_mean), sigma=current_vol) - 1
            else:
                simulated_returns = np.random.multivariate_normal(current_mean, adjusted_cov_matrix)
            
            # Apply stress shocks if applicable
            if stress_shocks and t == stress_period:
                for idx, ticker in enumerate(available_selected):
                    simulated_returns[idx] += stress_shocks.get(ticker, 0)
            
            portfolio_return = np.dot(simulated_returns, weights)
            simulated_prices.append(simulated_prices[-1] * (1 + portfolio_return))
        cumulative_return = simulated_prices[-1] - 1
        return cumulative_return

    portfolio_returns = Parallel(n_jobs=-1)(
        delayed(simulate)() for _ in range(num_simulations)
    )

    return np.array(portfolio_returns)
    
def backtest(weights, prices, rebalance_freq='M', broker_fee=0.0, debug=False):
    try:
        # Calculate returns and drop any NaN values
        returns = prices.pct_change().dropna()
        if returns.empty:
            st.error("Returns data is empty after calculating percentage changes.")
            return pd.Series(dtype=float), pd.Series(dtype=float)
        
        # Calculate portfolio returns
        portfolio_returns = returns.dot(weights)
        if not isinstance(portfolio_returns, pd.Series):
            portfolio_returns = portfolio_returns.squeeze()

        # Identify rebalancing dates
        if rebalance_freq == 'D':
            rebalance_dates = returns.index
        else:
            rebalance_dates = returns.resample(rebalance_freq).last().dropna().index

        # Debug: Show rebalancing dates
        if debug:
            st.write(f"Rebalancing Dates: {rebalance_dates.tolist()}")

        # Ensure rebalance_dates are in the portfolio_returns index
        valid_rebalance_dates = rebalance_dates.intersection(portfolio_returns.index)
        if debug:
            st.write(f"Valid Rebalancing Dates: {valid_rebalance_dates.tolist()}")

        # Apply broker fees on rebalancing dates
        if not valid_rebalance_dates.empty:
            portfolio_returns.loc[valid_rebalance_dates] -= broker_fee / 100  # Convert to decimal
        else:
            st.warning("No valid rebalancing dates found within the returns data.")

        # Calculate cumulative returns
        cum_returns = (1 + portfolio_returns).cumprod()
        if cum_returns.empty:
            st.error("Cumulative returns are empty. Check the data and allocations.")
            return pd.Series(dtype=float), pd.Series(dtype=float)

        # Debug: Show cumulative returns
        if debug:
            st.write("Cumulative Returns:")
            st.write(cum_returns)

        return portfolio_returns, cum_returns
    except Exception as e:
        st.error(f"Error during backtesting: {e}")
        return pd.Series(dtype=float), pd.Series(dtype=float)
    
def optimize_portfolio(
    returns, 
    benchmark_returns=None, 
    objectives=['sharpe'], 
    rf=0.02, 
    max_weight=1.0, 
    min_weight=0.0, 
    target_return=None
):
    """
    Optimize a portfolio based on specified objectives.

    Parameters:
    - returns (pd.DataFrame or np.ndarray): Historical returns of assets.
    - benchmark_returns (pd.Series or np.ndarray, optional): Returns of the benchmark index.
    - objectives (list of str): Objectives to optimize. 
        Options include 'sharpe', 'min_variance', 'max_return', 'min_drawdown', 'maximize_alpha', 'minimize_beta'.
    - rf (float): Risk-free rate for alpha calculation.
    - max_weight (float): Maximum weight per asset.
    - min_weight (float): Minimum weight per asset.
    - target_return (float, optional): Target return for the portfolio.

    Returns:
    - np.ndarray: Optimized asset weights or None if optimization fails.
    """
    
    # Define helper functions (ensure these are implemented elsewhere)
    def calculate_beta(portfolio_returns, benchmark_returns):
        covariance = np.cov(portfolio_returns, benchmark_returns)[0, 1]
        variance = np.var(benchmark_returns)
        return covariance / variance if variance != 0 else 0

    def calculate_var(portfolio_returns, confidence_level=0.95):
        if not isinstance(portfolio_returns, np.ndarray):
            portfolio_returns = np.array(portfolio_returns)
        return np.percentile(portfolio_returns, (1 - confidence_level) * 100)

    def calculate_cvar(portfolio_returns, confidence_level=0.95):
        var = calculate_var(portfolio_returns, confidence_level)
        portfolio_returns = np.array(portfolio_returns)
        return portfolio_returns[portfolio_returns <= var].mean()

    # Define individual objective functions
    def sharpe_ratio(weights):
        portfolio_return = np.dot(returns.mean(), weights) * 252
        portfolio_volatility = np.sqrt(np.dot(weights.T, np.dot(returns.cov() * 252, weights)))
        return -portfolio_return / portfolio_volatility if portfolio_volatility > 1e-6 else 0  # Negative for maximization

    def min_variance(weights):
        return np.dot(weights.T, np.dot(returns.cov() * 252, weights))

    def max_return(weights):
        return -np.dot(returns.mean(), weights) * 252  # Negative for maximization

    def min_drawdown(weights):
        # Simplistic drawdown minimization using expected shortfall (CVaR)
        portfolio_returns = returns.dot(weights)
        cvar = calculate_cvar(portfolio_returns, 0.95)
        return cvar  # Minimizing CVaR as a proxy for drawdown

    # Initialize list of objective functions
    objective_functions = []
    
    # Iterate through selected objectives and append corresponding functions
    for obj in objectives:
        if obj == 'sharpe':
            objective_functions.append(sharpe_ratio)
        elif obj == 'min_variance':
            objective_functions.append(min_variance)
        elif obj == 'max_return':
            objective_functions.append(max_return)
        elif obj == 'min_drawdown':
            objective_functions.append(min_drawdown)
        elif obj == 'maximize_alpha':
            if benchmark_returns is None:
                st.error("benchmark_returns must be provided for 'maximize_alpha' objective.")
                return None
            
            benchmark_return = benchmark_returns.mean() * 252  # Assuming daily returns annualized
            def maximize_alpha(weights):
                portfolio_return = np.dot(returns.mean(), weights) * 252
                portfolio_volatility = np.sqrt(np.dot(weights.T, np.dot(returns.cov() * 252, weights)))
                portfolio_returns = returns.dot(weights)
                beta = calculate_beta(portfolio_returns, benchmark_returns)
                alpha = portfolio_return - (rf + beta * (benchmark_return - rf))
                return -alpha  # Negative for maximization

            objective_functions.append(maximize_alpha)
        elif obj == 'minimize_beta':
            if benchmark_returns is None:
                st.error("benchmark_returns must be provided for 'minimize_beta' objective.")
                return None
            
            def minimize_beta(weights):
                portfolio_returns = returns.dot(weights)
                beta = calculate_beta(portfolio_returns, benchmark_returns)
                return beta

            objective_functions.append(minimize_beta)
        else:
            st.error(f"Invalid optimization objective: {obj}")
            return None

    # Composite objective function: weighted sum of individual objectives
    def composite_objective(weights):
        return sum(fn(weights) for fn in objective_functions)

    # Define constraints
    constraints = [
        {'type': 'eq', 'fun': lambda x: np.sum(x) - 1},
    ]

    if target_return is not None:
        def target_return_constraint(x):
            return np.dot(x, returns.mean()) * 252 - target_return
        constraints.append({
            'type': 'ineq',
            'fun': target_return_constraint
        })

    # Define bounds for weights
    bounds = tuple((min_weight, max_weight) for _ in range(returns.shape[1]))
    
    # Initial guess (equally distributed weights)
    initial_guess = np.array([1.0 / returns.shape[1]] * returns.shape[1])

    # Perform optimization
    result = minimize(
        composite_objective, 
        initial_guess, 
        method='SLSQP', 
        bounds=bounds, 
        constraints=constraints
    )
    
    if result.success:
        optimized_weights = result.x
        portfolio_volatility = np.sqrt(np.dot(optimized_weights.T, np.dot(returns.cov() * 252, optimized_weights)))
        
        if portfolio_volatility <= 1e-6:
            st.error("Optimized portfolio has near-zero volatility. Optimization constraints may be too restrictive.")
            return None
        
        return optimized_weights
    else:
        st.error("Optimization failed. Try adjusting your constraints or target return.")
        return None

fix the drawdown is empty when portoflio vs portflio