import os
import json
import logging
import traceback
from datetime import datetime
from flask import Flask, request, jsonify
from flask_cors import CORS
import yfinance as yf
import pandas as pd
import numpy as np
import empyrical as ep

app = Flask(__name__)
CORS(app)  # Enable CORS for all routes

# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# Constants
PORTFOLIO_DIR = 'portfolios'
CACHE_DIR = 'cache'
if not os.path.exists(PORTFOLIO_DIR):
    os.makedirs(PORTFOLIO_DIR)
if not os.path.exists(CACHE_DIR):
    os.makedirs(CACHE_DIR)

# Utility Functions
def get_cache_key(ticker, start_date):
    """Generate a cache key based on ticker and start date."""
    # Convert datetime to string format safe for filenames
    if isinstance(start_date, pd.Timestamp):
        start_date = start_date.strftime('%Y%m%d')
    elif isinstance(start_date, str):
        start_date = pd.to_datetime(start_date).strftime('%Y%m%d')
    return f"{ticker}_{start_date}.json"

def load_from_cache(cache_key):
    """Load data from cache if available."""
    try:
        cache_path = os.path.join(CACHE_DIR, cache_key)
        if os.path.exists(cache_path):
            data = pd.read_json(cache_path)
            # Convert string index back to datetime
            data.index = pd.to_datetime(data.index)
            if 'Adj Close' not in data.columns:
                logger.warning(f"Invalid cache data structure in {cache_key}")
                return None
            logger.info(f"Successfully loaded cache from {cache_path}")
            return data
    except Exception as e:
        logger.error(f"Failed to load cache for {cache_key}: {str(e)}")
    return None

def save_to_cache(cache_key, data):
    """Save data to cache."""
    try:
        cache_path = os.path.join(CACHE_DIR, cache_key)
        # Convert datetime index to string format before saving
        data.index = data.index.strftime('%Y-%m-%d')
        data.to_json(cache_path)
        logger.info(f"Successfully cached data to {cache_path}")
    except Exception as e:
        logger.error(f"Failed to save cache for {cache_key}: {str(e)}")

def clean_for_json(obj):
    """Recursively clean objects to make them JSON serializable."""
    if isinstance(obj, (np.integer, np.floating)):
        return float(obj) if not (np.isnan(obj) or np.isinf(obj)) else None
    elif isinstance(obj, dict):
        return {k: clean_for_json(v) for k, v in obj.items()}
    elif isinstance(obj, (list, tuple)):
        return [clean_for_json(x) for x in obj]
    elif isinstance(obj, pd.Series):
        return [clean_for_json(x) for x in obj.values]
    elif isinstance(obj, pd.DataFrame):
        return {col: clean_for_json(obj[col]) for col in obj.columns}
    elif isinstance(obj, np.ndarray):
        return [clean_for_json(x) for x in obj.flatten()]
    elif pd.isna(obj):
        return None
    return obj

def validate_ticker(ticker):
    """
    Validate a stock ticker symbol using yfinance.
    Args:
        ticker (str): The stock ticker symbol.
    Returns:
        bool: True if valid, False otherwise.
    """
    try:
        ticker = ticker.strip().upper()
        if not ticker:
            return False
        stock = yf.Ticker(ticker)
        info = stock.info
        return bool(info)
    except Exception as e:
        logger.error(f"Ticker validation failed for {ticker}: {str(e)}")
        return False

def fetch_stock_data(ticker, start_date):
    """
    Fetch historical stock data from Yahoo Finance with caching.
    Args:
        ticker (str): Stock ticker symbol.
        start_date (str): Start date in 'YYYY-MM-DD' format.
    Returns:
        pd.DataFrame: Historical stock data.
    """
    try:
        # Format the cache key
        cache_key = f"{ticker}_{pd.to_datetime(start_date).strftime('%Y%m%d')}.json"
        cached_data = load_from_cache(cache_key)

        if cached_data is not None and not cached_data.empty:
            logger.info(f"Using cached data for {ticker}")
            return cached_data

        # If cache miss or invalid cache, fetch new data
        logger.info(f"Fetching new data for {ticker}")
        stock_data = yf.download(
            ticker, 
            start=start_date, 
            progress=False,
            auto_adjust=False  # Changed to False to get Adj Close column
        )

        if not stock_data.empty:
            # Ensure we have the Adj Close column
            if 'Adj Close' not in stock_data.columns:
                logger.error(f"Missing Adj Close column for {ticker}")
                return None

            # Save to cache
            try:
                save_to_cache(cache_key, stock_data)
                logger.info(f"Data fetched and cached for {ticker} from {start_date}")
            except Exception as e:
                logger.error(f"Failed to cache data for {ticker}: {str(e)}")

            return stock_data
        else:
            logger.warning(f"No data fetched for {ticker} from {start_date}")
            return None

    except Exception as e:
        logger.error(f"Failed to fetch data for {ticker}: {str(e)}")
        return None

def calculate_portfolio_metrics(returns, benchmark_returns=None):
    """Calculate portfolio metrics using the empyrical library."""
    metrics = {}
    logger.info("Starting portfolio metrics calculation...")
    
    try:
        # Ensure returns is a pandas Series and contains numeric values
        if isinstance(returns, pd.DataFrame):
            returns = returns.iloc[:, 0]  # Take first column if DataFrame
        
        # Convert returns to numeric, replacing any non-numeric values with NaN
        returns = pd.to_numeric(returns, errors='coerce').dropna()
        
        # Store returns for chart
        metrics['returns'] = returns.tolist()
        
        # Basic portfolio metrics
        metrics['start_balance'] = 10000.0
        metrics['end_balance'] = float(metrics['start_balance'] * (1 + returns).prod())
        metrics['total_return'] = float((metrics['end_balance'] / metrics['start_balance']) - 1)
        
        # Risk-free rate (2% annual)
        risk_free_rate = 0.02 / 252  # Daily risk-free rate
        
        # Calculate base metrics using empyrical
        try:
            # Annualized return calculation
            total_years = len(returns) / 252  # Assuming 252 trading days per year
            metrics['cagr'] = float(((1 + metrics['total_return']) ** (1 / total_years)) - 1)
            
            # Volatility calculation (annualized)
            daily_std = float(returns.std())
            metrics['annual_volatility'] = float(daily_std * np.sqrt(252))
            
            # Other risk metrics
            metrics['sharpe_ratio'] = float(ep.sharpe_ratio(returns, risk_free=risk_free_rate))
            metrics['sortino_ratio'] = float(ep.sortino_ratio(returns, required_return=risk_free_rate))
            metrics['max_drawdown'] = float(ep.max_drawdown(returns))
            
            logger.info(f"Base metrics calculated: CAGR={metrics['cagr']:.2%}, Vol={metrics['annual_volatility']:.2%}")
        except Exception as e:
            logger.error(f"Error calculating base metrics: {str(e)}")
            metrics.update({
                'cagr': 0.0,
                'annual_volatility': 0.0,
                'sharpe_ratio': 0.0,
                'sortino_ratio': 0.0,
                'max_drawdown': 0.0
            })
        
        # Calculate drawdowns
        try:
            drawdowns = calculate_drawdown_periods(returns)
            metrics['drawdowns'] = drawdowns
            logger.info(f"Found {len(drawdowns)} significant drawdown periods")
        except Exception as e:
            logger.error(f"Error calculating drawdowns: {str(e)}")
            metrics['drawdowns'] = []

        # If benchmark_returns is provided, calculate relative metrics
        if benchmark_returns is not None and len(benchmark_returns) > 0:
            try:
                # Align returns for benchmark calculations
                common_dates = returns.index.intersection(benchmark_returns.index)
                aligned_returns = returns[common_dates]
                aligned_benchmark = benchmark_returns[common_dates]
                
                # Calculate alpha and beta
                try:
                    metrics['alpha'] = float(ep.alpha(
                        aligned_returns, 
                        aligned_benchmark,
                        risk_free=risk_free_rate,
                        period='daily'
                    ))
                    metrics['beta'] = float(ep.beta(
                        aligned_returns,
                        aligned_benchmark,
                        risk_free=risk_free_rate
                    ))
                except Exception as e:
                    logger.error(f"Error calculating alpha/beta: {str(e)}")
                    metrics['alpha'] = 0.0
                    metrics['beta'] = 1.0
                
                # Calculate information ratio and tracking error
                try:
                    excess_returns = aligned_returns - aligned_benchmark
                    annualized_excess_return = excess_returns.mean() * 252
                    tracking_error = excess_returns.std() * np.sqrt(252)
                    
                    if tracking_error > 0:
                        metrics['information_ratio'] = float(annualized_excess_return / tracking_error)
                    else:
                        metrics['information_ratio'] = 0.0
                    
                    metrics['tracking_error'] = float(tracking_error)
                except Exception as e:
                    logger.error(f"Error calculating IR/TE: {str(e)}")
                    metrics['information_ratio'] = 0.0
                    metrics['tracking_error'] = 0.0
                
                # Calculate R-squared
                try:
                    correlation = np.corrcoef(aligned_returns, aligned_benchmark)[0, 1]
                    metrics['r_squared'] = float(correlation ** 2)
                except Exception as e:
                    logger.error(f"Error calculating R-squared: {str(e)}")
                    metrics['r_squared'] = 1.0
                
                # Calculate capture ratios
                try:
                    up_periods = aligned_benchmark > 0
                    down_periods = aligned_benchmark < 0
                    
                    if up_periods.any():
                        up_capture = (aligned_returns[up_periods].mean() / 
                                      aligned_benchmark[up_periods].mean())
                        metrics['up_capture'] = float(up_capture)
                    else:
                        metrics['up_capture'] = 1.0
                        
                    if down_periods.any():
                        down_capture = (aligned_returns[down_periods].mean() / 
                                        aligned_benchmark[down_periods].mean())
                        metrics['down_capture'] = float(down_capture)
                    else:
                        metrics['down_capture'] = 1.0
                except Exception as e:
                    logger.error(f"Error calculating capture ratios: {str(e)}")
                    metrics['up_capture'] = 1.0
                    metrics['down_capture'] = 1.0
                
            except Exception as e:
                logger.error(f"Error in benchmark calculations: {str(e)}")
                logger.error(traceback.format_exc())
                metrics.update({
                    'alpha': 0.0,
                    'beta': 1.0,
                    'information_ratio': 0.0,
                    'tracking_error': 0.0,
                    'r_squared': 1.0,
                    'up_capture': 1.0,
                    'down_capture': 1.0
                })
        else:
            # Set default values for relative metrics when no benchmark
            metrics.update({
                'alpha': 0.0,
                'beta': 1.0,
                'information_ratio': 0.0,
                'tracking_error': 0.0,
                'r_squared': 1.0,
                'up_capture': 1.0,
                'down_capture': 1.0
            })

        return metrics
    except Exception as e:
        logger.error(f"Error in calculate_portfolio_metrics: {str(e)}")
        return {
            'cagr': 0.0,
            'annual_volatility': 0.0,
            'sharpe_ratio': 0.0,
            'sortino_ratio': 0.0,
            'max_drawdown': 0.0,
            'alpha': 0.0,
            'beta': 1.0,
            'information_ratio': 0.0,
            'tracking_error': 0.0,
            'r_squared': 1.0,
            'up_capture': 1.0,
            'down_capture': 1.0,
            'drawdowns': []
        }

def calculate_correlation_matrix(portfolio_data):
    """Calculate correlation matrices for the portfolio."""
    try:
        # Calculate returns
        returns = portfolio_data.pct_change().dropna()
        
        # Calculate Pearson correlation
        pearson_matrix = returns.corr(method='pearson')
        pearson_values = pearson_matrix.values[np.triu_indices_from(pearson_matrix.values, k=1)]
        
        # Calculate Spearman correlation
        spearman_matrix = returns.corr(method='spearman')
        spearman_values = spearman_matrix.values[np.triu_indices_from(spearman_matrix.values, k=1)]
        
        # Convert matrices to dictionaries
        pearson_dict = {ticker: pearson_matrix[ticker].to_dict() for ticker in pearson_matrix.index}
        spearman_dict = {ticker: spearman_matrix[ticker].to_dict() for ticker in spearman_matrix.index}
        
        return {
            'pearson': {
                'matrix': pearson_dict,
                'avg': float(np.mean(pearson_values)),
                'min': float(np.min(pearson_values)),
                'max': float(np.max(pearson_values))
            },
            'spearman': {
                'matrix': spearman_dict,
                'avg': float(np.mean(spearman_values)),
                'min': float(np.min(spearman_values)),
                'max': float(np.max(spearman_values))
            }
        }
    except Exception as e:
        logger.error(f"Error in calculate_correlation_matrix: {str(e)}", exc_info=True)
        return None

def calculate_drawdown_periods(returns):
    """
    Identify drawdown periods in the portfolio.
    Args:
        returns (pd.Series): Daily returns of the portfolio.
    Returns:
        list: A list of drawdown periods with details.
    """
    try:
        # Calculate cumulative returns and high water mark
        cumulative = (1 + returns).cumprod()
        running_max = cumulative.expanding().max()
        drawdowns = (cumulative - running_max) / running_max

        # Find significant drawdown periods (more than 5%)
        DRAWDOWN_THRESHOLD = -0.05  # 5% threshold
        significant_drawdowns = []
        in_drawdown = False
        peak_idx = None
        trough_idx = None

        for date in drawdowns.index:
            current_drawdown = drawdowns[date]
            
            # Start of drawdown
            if not in_drawdown and current_drawdown < DRAWDOWN_THRESHOLD:
                in_drawdown = True
                # Find the actual peak before this drawdown
                peak_idx = running_max[:date].idxmax()
                trough_idx = date
            
            # Update trough if in drawdown
            elif in_drawdown and current_drawdown < drawdowns[trough_idx]:
                trough_idx = date
            
            # Check for recovery
            elif in_drawdown and cumulative[date] >= cumulative[peak_idx]:
                # Convert indices to datetime for calculation
                peak_date = pd.to_datetime(peak_idx)
                trough_date = pd.to_datetime(trough_idx)
                recovery_date = pd.to_datetime(date)
                
                drawdown_info = {
                    'start': peak_date.strftime('%Y-%m-%d'),
                    'end': trough_date.strftime('%Y-%m-%d'),
                    'recovery': recovery_date.strftime('%Y-%m-%d'),
                    'depth': float((cumulative[trough_idx] - cumulative[peak_idx]) / cumulative[peak_idx]),
                    'length': (trough_date - peak_date).days,
                    'recovery_time': (recovery_date - peak_date).days
                }
                significant_drawdowns.append(drawdown_info)
                
                # Reset for next drawdown
                in_drawdown = False
                peak_idx = None
                trough_idx = None

        # Handle ongoing drawdown
        if in_drawdown and peak_idx is not None and trough_idx is not None:
            # Convert indices to datetime for calculation
            peak_date = pd.to_datetime(peak_idx)
            trough_date = pd.to_datetime(trough_idx)
            
            drawdown_info = {
                'start': peak_date.strftime('%Y-%m-%d'),
                'end': trough_date.strftime('%Y-%m-%d'),
                'recovery': None,
                'depth': float((cumulative[trough_idx] - cumulative[peak_idx]) / cumulative[peak_idx]),
                'length': (trough_date - peak_date).days,
                'recovery_time': None
            }
            significant_drawdowns.append(drawdown_info)

        # Sort drawdowns by depth (most severe first)
        significant_drawdowns.sort(key=lambda x: x['depth'])

        logger.info(f"Found {len(significant_drawdowns)} significant drawdown periods")
        return significant_drawdowns

    except Exception as e:
        logger.error(f"Error in calculate_drawdown_periods: {str(e)}", exc_info=True)
        return []

def calculate_rebalancing_impact(portfolio_data, weights, transaction_cost):
    """Calculate the impact of different rebalancing strategies."""
    try:
        strategies = {
            'null': None,  # Buy and hold
            'ME': 'M',     # Monthly
            'QE': 'Q',     # Quarterly
            'YE': 'Y'      # Yearly
        }
        
        results = {}
        
        # Calculate returns
        returns = portfolio_data.pct_change().fillna(0)
        
        for strategy_name, freq in strategies.items():
            # Initialize portfolio
            portfolio_value = 10000  # Start with $10,000
            current_weights = weights.copy()
            current_holdings = portfolio_value * current_weights
            total_costs = 0
            rebalance_count = 0
            portfolio_values = []
            daily_returns = []
            
            # For buy and hold, just calculate the portfolio value over time
            if freq is None:
                for date, daily_returns_row in returns.iterrows():
                    # Update holdings based on daily returns
                    current_holdings *= (1 + daily_returns_row)
                    portfolio_value = current_holdings.sum()
                    portfolio_values.append(portfolio_value)
                    
                    # Calculate daily return
                    if len(portfolio_values) > 1:
                        daily_return = (portfolio_value / portfolio_values[-2]) - 1
                    else:
                        daily_return = 0
                    daily_returns.append(daily_return)
                    
                    # Update current weights
                    current_weights = current_holdings / portfolio_value
            else:
                # For rebalancing strategies
                last_rebalance = None
                
                for date, daily_returns_row in returns.iterrows():
                    # Update holdings based on daily returns
                    current_holdings *= (1 + daily_returns_row)
                    portfolio_value = current_holdings.sum()
                    current_weights = current_holdings / portfolio_value
                    
                    # Calculate daily return before rebalancing
                    if len(portfolio_values) > 0:
                        daily_return = (portfolio_value / portfolio_values[-1]) - 1
                    else:
                        daily_return = 0
                    
                    # Check if rebalancing is needed
                    # Convert dates to periods without timezone info
                    date_period = pd.Timestamp(date).to_period(freq)
                    last_period = pd.Timestamp(last_rebalance).to_period(freq) if last_rebalance is not None else None
                    
                    if last_period is None or date_period != last_period:
                        # Calculate trades needed
                        target_holdings = portfolio_value * weights
                        trades = target_holdings - current_holdings
                        
                        # Calculate transaction costs
                        total_cost = abs(trades).sum() * transaction_cost
                        
                        # Only rebalance if the benefit outweighs the cost
                        weight_drift = np.abs(current_weights - weights).sum()
                        if weight_drift > transaction_cost * 2:
                            # Apply transaction costs
                            portfolio_value -= total_cost
                            total_costs += total_cost
                            
                            # Update holdings and record rebalancing
                            current_holdings = portfolio_value * weights
                            last_rebalance = date
                            rebalance_count += 1
                            
                            # Adjust daily return for transaction costs
                            daily_return -= total_cost / portfolio_values[-1] if portfolio_values else 0
                    
                    portfolio_values.append(portfolio_value)
                    daily_returns.append(daily_return)
            
            # Calculate strategy metrics
            daily_returns_series = pd.Series(daily_returns, index=returns.index)
            
            metrics = calculate_portfolio_metrics(daily_returns_series)
            metrics.update({
                'final_value': portfolio_values[-1],
                'total_return': (portfolio_values[-1] / 10000) - 1,
                'rebalance_count': rebalance_count,
                'total_cost': total_costs,
                'cost_percentage': total_costs / portfolio_values[-1],
                'annualized_return': metrics['cagr'],
                'volatility': metrics['annual_volatility']
            })
            
            results[strategy_name] = metrics
        
        return results
        
    except Exception as e:
        logger.error(f"Error in calculate_rebalancing_impact: {str(e)}", exc_info=True)
        return {}

# API Routes
@app.route('/api/compare', methods=['POST', 'OPTIONS'])
def compare_portfolios():
    """Compare multiple portfolios."""
    # Handle CORS preflight request
    if request.method == 'OPTIONS':
        response = app.make_default_options_response()
        response.headers['Access-Control-Allow-Methods'] = 'POST'
        response.headers['Access-Control-Allow-Headers'] = 'Content-Type'
        return response

    try:
        logger.info("Starting portfolio comparison...")
        data = request.get_json()
        logger.info(f"Received request data: {json.dumps(data, indent=2)}")
        
        portfolios = data.get('portfolios', [])
        start_date = data.get('startDate')
        transaction_cost = min(data.get('transactionCost', 0.001), 0.01)  # Cap at 1%

        if not portfolios:
            return jsonify({
                'success': False,
                'error': 'No portfolios provided for comparison'
            }), 400

        # Initialize results dictionary
        comparison_results = {}
        
        start_date = pd.to_datetime(start_date)
        all_data = pd.DataFrame()
        
        # Collect all unique tickers
        unique_tickers = set()
        for portfolio in portfolios:
            portfolio_stocks = portfolio.get('portfolio', [])
            unique_tickers.update([stock['ticker'] for stock in portfolio_stocks])

        logger.info(f"Fetching data for {len(unique_tickers)} unique tickers...")

        # Fetch data for all tickers
        for ticker in unique_tickers:
            stock_data = fetch_stock_data(ticker, start_date)
            if stock_data is not None and not stock_data.empty:
                all_data[ticker] = stock_data['Adj Close']
            else:
                return jsonify({
                    'success': False,
                    'error': f'Failed to fetch data for {ticker}'
                }), 400

        # Align all data to common dates
        all_data = all_data.dropna()
        if len(all_data) < 252:  # Approximately one year of trading days
            return jsonify({
                'success': False,
                'error': 'Insufficient historical data. Need at least 252 data points.'
            }), 400

        # Calculate metrics for each portfolio
        for portfolio in portfolios:
            portfolio_name = portfolio.get('name', 'Unnamed Portfolio')
            portfolio_stocks = portfolio.get('portfolio', [])
            
            # Calculate portfolio returns
            weights = np.array([float(stock['weight']) for stock in portfolio_stocks])
            tickers = [stock['ticker'] for stock in portfolio_stocks]
            portfolio_data = all_data[tickers]
            portfolio_returns = portfolio_data.pct_change().fillna(0)
            weighted_returns = (portfolio_returns * weights).sum(axis=1)
            
            # Calculate metrics
            metrics = calculate_portfolio_metrics(weighted_returns)
            
            # Calculate rebalancing impact
            rebalancing_results = calculate_rebalancing_impact(
                portfolio_data, 
                weights,
                transaction_cost
            )
            
            comparison_results[portfolio_name] = {
                'metrics': metrics,
                'rebalancing': rebalancing_results,
                'portfolio': portfolio_stocks
            }

        # Convert dates to string format for JSON serialization
        dates = all_data.index.strftime('%Y-%m-%d').tolist()

        return jsonify({
            'success': True,
            'comparison_results': comparison_results,
            'dates': dates,
            'data_quality': {
                'num_data_points': len(all_data),
                'date_range': {
                    'start': all_data.index[0].strftime('%Y-%m-%d'),
                    'end': all_data.index[-1].strftime('%Y-%m-%d')
                },
                'missing_values': False,
                'aligned_data': True
            }
        })

    except Exception as e:
        logger.error(f"Error in compare_portfolios: {str(e)}")
        logger.error(traceback.format_exc())
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500

@app.route('/api/backtest', methods=['POST'])
def backtest():
    """
    Backtest the portfolio against historical data.
    Expects JSON payload with:
        - portfolio: list of stocks with 'ticker' and 'weight'
        - startDate: backtest start date in 'YYYY-MM-DD'
    """
    try:
        logger.info("Starting backtest...")
        data = request.get_json()
        logger.info(f"Received backtest data: {json.dumps(data, indent=2)}")
        
        portfolio = data.get('portfolio', [])
        start_date = data.get('startDate')
        
        if not portfolio:
            return jsonify({
                'success': False,
                'error': 'No portfolio provided for backtest'
            }), 400

        # Validate portfolio
        is_valid, error_msg = validate_portfolio(portfolio)
        if not is_valid:
            return jsonify({
                'success': False,
                'error': f"Invalid portfolio: {error_msg}"
            }), 400

        start_date = pd.to_datetime(start_date)
        portfolio_data = pd.DataFrame()

        # Fetch data for portfolio stocks
        logger.info("Fetching portfolio data...")
        for stock in portfolio:
            ticker = stock['ticker']
            stock_data = fetch_stock_data(ticker, start_date)
            if stock_data is not None and not stock_data.empty:
                portfolio_data[ticker] = stock_data['Adj Close']
                logger.info(f"Data fetched for {ticker}, shape: {stock_data.shape}")
            else:
                return jsonify({
                    'success': False,
                    'error': f'No data available for {ticker}'
                }), 400

        # Fetch or define benchmark
        benchmark = 'SPY'  # Default benchmark
        benchmark_data = fetch_stock_data(benchmark, start_date)
        if benchmark_data is None or benchmark_data.empty:
            return jsonify({
                'success': False,
                'error': f'No data available for benchmark {benchmark}'
            }), 400

        # Calculate metrics
        portfolio_returns = portfolio_data.pct_change().dropna()
        benchmark_returns = benchmark_data['Adj Close'].pct_change().dropna().reindex(portfolio_returns.index)
        benchmark_returns = benchmark_returns.fillna(0)

        portfolio_metrics = calculate_portfolio_metrics(portfolio_returns)
        benchmark_metrics = calculate_portfolio_metrics(benchmark_returns)

        # Calculate cumulative portfolio and benchmark values
        initial_investment = portfolio_metrics.get('start_balance', 10000.0)
        portfolio_cumulative = (1 + portfolio_returns).cumprod() * initial_investment
        benchmark_cumulative = (1 + benchmark_returns).cumprod() * initial_investment

        # Calculate drawdowns
        portfolio_drawdowns = calculate_drawdown_periods(portfolio_returns)
        benchmark_drawdowns = calculate_drawdown_periods(benchmark_returns)

        # Prepare response with data quality information
        common_dates = portfolio_returns.index
        response_data = {
            'success': True,
            'portfolio_metrics': clean_for_json(portfolio_metrics),
            'benchmark_metrics': clean_for_json(benchmark_metrics),
            'historical_performance': {
                'dates': common_dates.strftime('%Y-%m-%d').tolist(),
                'portfolio_values': portfolio_cumulative.tolist(),
                'benchmark_values': benchmark_cumulative.tolist(),
                'initial_investment': initial_investment
            },
            'drawdowns': {
                'portfolio_drawdowns': portfolio_drawdowns,
                'benchmark_drawdowns': benchmark_drawdowns
            },
            'data_quality': {
                'num_data_points': len(common_dates),
                'date_range': {
                    'start': common_dates[0].strftime('%Y-%m-%d'),
                    'end': common_dates[-1].strftime('%Y-%m-%d')
                },
                'missing_values': False,
                'aligned_data': True
            }
        }

        logger.info("Backtest completed successfully")
        return jsonify(response_data)

    except Exception as e:
        logger.error(f"Error in backtest: {str(e)}", exc_info=True)
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500

@app.route('/api/correlation', methods=['POST'])
def get_correlation():
    """
    Calculate and return correlation matrices for the portfolio.
    Expects JSON payload with:
        - portfolio: list of stocks with 'ticker' and 'weight'
    """
    try:
        logger.info("Starting correlation analysis...")
        data = request.get_json()
        logger.info(f"Received correlation data: {json.dumps(data, indent=2)}")
        
        portfolio = data.get('portfolio', [])
        if not portfolio or len(portfolio) < 2:
            return jsonify({
                'success': False,
                'error': 'Portfolio must contain at least two stocks for correlation analysis'
            }), 400

        # Validate portfolio
        is_valid, error_msg = validate_portfolio(portfolio)
        if not is_valid:
            return jsonify({
                'success': False,
                'error': f"Invalid portfolio: {error_msg}"
            }), 400

        # Fetch data for portfolio stocks
        portfolio_tickers = [stock['ticker'] for stock in portfolio]
        portfolio_data = pd.DataFrame()

        logger.info("Fetching portfolio data for correlation analysis...")
        for ticker in portfolio_tickers:
            stock_data = fetch_stock_data(ticker, start_date='2000-01-01')  # Using a fixed start date
            if stock_data is not None and not stock_data.empty:
                portfolio_data[ticker] = stock_data['Adj Close']
                logger.info(f"Data fetched for {ticker}, shape: {stock_data.shape}")
            else:
                return jsonify({
                    'success': False,
                    'error': f'No data available for {ticker}'
                }), 400

        # Calculate correlation matrices
        correlations = calculate_correlation_matrix(portfolio_data)

        return jsonify({
            'success': True,
            'correlations': correlations
        })

    except Exception as e:
        logger.error(f"Error in get_correlation: {str(e)}", exc_info=True)
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500

@app.route('/api/attribution', methods=['POST'])
def get_attribution():
    """
    Perform attribution analysis on the portfolio.
    Expects JSON payload with:
        - portfolio: list of stocks with 'ticker' and 'weight'
        - startDate: analysis start date in 'YYYY-MM-DD'
        - benchmark: benchmark ticker symbol
    """
    try:
        logger.info("Starting attribution analysis...")
        data = request.get_json()
        logger.info(f"Received attribution data: {json.dumps(data, indent=2)}")
        
        portfolio = data.get('portfolio', [])
        start_date = data.get('startDate')
        benchmark = data.get('benchmark', 'SPY')
        
        if not portfolio:
            return jsonify({
                'success': False,
                'error': 'No portfolio provided'
            }), 400

        # Validate portfolio
        is_valid, error_msg = validate_portfolio(portfolio)
        if not is_valid:
            return jsonify({
                'success': False,
                'error': f"Invalid portfolio: {error_msg}"
            }), 400

        start_date = pd.to_datetime(start_date)
        portfolio_tickers = [stock['ticker'] for stock in portfolio]
        portfolio_weights = np.array([float(stock['weight']) for stock in portfolio])

        # Fetch data for portfolio stocks
        logger.info("Fetching portfolio data for attribution analysis...")
        portfolio_data = pd.DataFrame()
        for ticker in portfolio_tickers:
            stock_data = fetch_stock_data(ticker, start_date)
            if stock_data is not None and not stock_data.empty:
                portfolio_data[ticker] = stock_data['Adj Close']
                logger.info(f"Data fetched for {ticker}, shape: {stock_data.shape}")
            else:
                return jsonify({
                    'success': False,
                    'error': f'No data available for {ticker}'
                }), 400

        # Fetch benchmark data
        logger.info(f"Fetching benchmark data for {benchmark}")
        benchmark_data = fetch_stock_data(benchmark, start_date)
        if benchmark_data is None or benchmark_data.empty:
            return jsonify({
                'success': False,
                'error': f'No data available for benchmark {benchmark}'
            }), 400

        return jsonify({
            'success': True,
        })

    except Exception as e:
        logger.error(f"Error in get_attribution: {str(e)}", exc_info=True)
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500

# Ensure no duplicate route definitions exist
# Removed duplicate delete_portfolio route; ensured unique endpoint names

@app.after_request
def after_request(response):
    """Add CORS headers to all responses"""
    response.headers.add('Access-Control-Allow-Origin', '*')
    response.headers.add('Access-Control-Allow-Headers', 'Content-Type')
    response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS')
    return response

def validate_portfolio(portfolio):
    """
    Validate portfolio structure and weights.
    Args:
        portfolio (list): List of dictionaries containing ticker and weight
    Returns:
        tuple: (is_valid, error_message)
    """
    try:
        if not portfolio:
            return False, "Portfolio is empty"
            
        # Check for valid tickers and weights
        total_weight = 0
        for stock in portfolio:
            if not isinstance(stock, dict) or 'ticker' not in stock or 'weight' not in stock:
                return False, "Invalid portfolio structure"
                
            if not validate_ticker(stock['ticker']):
                return False, f"Invalid ticker: {stock['ticker']}"
            
            # Convert weight to float and validate
            try:
                # Weight should already be in decimal form (e.g., 0.5 for 50%)
                weight = float(stock['weight'])
                if weight <= 0 or weight > 1:
                    return False, f"Invalid weight for {stock['ticker']}: {weight * 100}%. Must be between 0% and 100%"
                total_weight += weight
            except (TypeError, ValueError):
                return False, f"Invalid weight format for {stock['ticker']}"
            
        # Check total weight (should be 1.0 for 100%)
        if abs(total_weight - 1.0) > 0.0001:  # Allow small rounding errors
            return False, f"Portfolio weights must sum to 100% (current: {total_weight * 100:.2f}%)"
            
        return True, None
        
    except Exception as e:
        logger.error(f"Portfolio validation error: {str(e)}")
        return False, str(e)

# Portfolio management routes
@app.route('/api/portfolio/load', methods=['GET'])
def load_portfolios():
    """Load all saved portfolios"""
    try:
        portfolios = []
        if os.path.exists(PORTFOLIO_DIR):
            for filename in os.listdir(PORTFOLIO_DIR):
                if filename.endswith('.json'):
                    file_path = os.path.join(PORTFOLIO_DIR, filename)
                    try:
                        with open(file_path, 'r') as f:
                            portfolio = json.load(f)
                            portfolio['filename'] = filename
                            # Add creation date from file metadata
                            created_at = os.path.getctime(file_path)
                            portfolio['created_at'] = datetime.fromtimestamp(created_at).isoformat()
                            portfolios.append(portfolio)
                    except Exception as e:
                        logger.error(f"Error loading portfolio {filename}: {str(e)}")
                        continue

        # Sort portfolios by creation date, newest first
        portfolios.sort(key=lambda x: x.get('created_at', ''), reverse=True)
        
        return jsonify({
            'success': True,
            'portfolios': portfolios
        })
    except Exception as e:
        logger.error(f"Error in load_portfolios: {str(e)}")
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500

@app.route('/api/portfolio/save', methods=['POST'])
def save_portfolio():
    """Save a portfolio"""
    try:
        data = request.get_json()
        if not data:
            return jsonify({
                'success': False,
                'error': 'No data provided'
            }), 400

        name = data.get('name')
        portfolio = data.get('portfolio', [])

        if not name:
            return jsonify({
                'success': False,
                'error': 'Portfolio name is required'
            }), 400

        # Log received portfolio data for debugging
        logger.info(f"Received portfolio data: {json.dumps(portfolio, indent=2)}")

        # Validate portfolio structure and weights
        is_valid, error_msg = validate_portfolio(portfolio)
        if not is_valid:
            return jsonify({
                'success': False,
                'error': error_msg
            }), 400

        # Create filename from name and timestamp
        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
        safe_name = "".join(x for x in name if x.isalnum() or x in (' ', '-', '_')).strip()
        filename = f"portfolio_{timestamp}_{safe_name}.json"
        file_path = os.path.join(PORTFOLIO_DIR, filename)

        # Save portfolio with weights in decimal form
        with open(file_path, 'w') as f:
            json.dump({
                'name': name,
                'portfolio': portfolio  # Weights should already be in decimal form
            }, f, indent=2)

        logger.info(f"Portfolio saved successfully: {filename}")
        return jsonify({
            'success': True,
            'filename': filename
        })
    except Exception as e:
        logger.error(f"Error in save_portfolio: {str(e)}")
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500

@app.route('/api/portfolio/delete/<path:filename>', methods=['DELETE', 'OPTIONS'])
def delete_portfolio_by_name(filename):
    """Delete a portfolio"""
    # Handle preflight request
    if request.method == 'OPTIONS':
        response = app.make_default_options_response()
        response.headers['Access-Control-Allow-Methods'] = 'DELETE'
        response.headers['Access-Control-Allow-Headers'] = 'Content-Type'
        return response

    try:
        if not filename.endswith('.json'):
            return jsonify({
                'success': False,
                'error': 'Invalid filename'
            }), 400

        file_path = os.path.join(PORTFOLIO_DIR, filename)
        if not os.path.exists(file_path):
            return jsonify({
                'success': False,
                'error': 'Portfolio not found'
            }), 404

        try:
            # Delete the file
            os.remove(file_path)
            logger.info(f"Successfully deleted portfolio: {filename}")
        except Exception as e:
            logger.error(f"Error deleting file {filename}: {str(e)}")
            return jsonify({
                'success': False,
                'error': f'Failed to delete file: {str(e)}'
            }), 500

        # Return updated portfolio list
        return load_portfolios()
    except Exception as e:
        logger.error(f"Error in delete_portfolio: {str(e)}")
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500

if __name__ == '__main__':
    app.run(debug=True)

ERROR:__main__:Error in compare_portfolios: 'Index' object has no attribute 'strftime'
ERROR:__main__:Traceback (most recent call last):
  File "C:\Users\calvi\Downloads\top\backend.py", line 646, in compare_portfolios
    dates = all_data.index.strftime('%Y-%m-%d').tolist()
AttributeError: 'Index' object has no attribute 'strftime'