from flask import Flask, request, jsonify
from flask_cors import CORS
import yfinance as yf
import pandas as pd
import numpy as np
from datetime import datetime, timedelta
import json
import os
from scipy import stats
from functools import lru_cache
import hashlib
import pickle
import empyrical as ep
import traceback
import io
import csv
import logging

app = Flask(__name__)
# Update CORS configuration to be more permissive for development
CORS(app, resources={
    r"/api/*": {
        "origins": ["http://localhost:5173", "http://127.0.0.1:5173"],
        "methods": ["GET", "POST", "OPTIONS"],
        "allow_headers": ["Content-Type"]
    }
})

# Configure Logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s %(levelname)s %(name)s %(threadName)s : %(message)s'
)
logger = logging.getLogger(__name__)

CACHE_DIR = 'cache'
PORTFOLIO_DIR = 'portfolios'
os.makedirs(CACHE_DIR, exist_ok=True)
os.makedirs(PORTFOLIO_DIR, exist_ok=True)

# Cache functions
def get_cache_key(ticker, start_date):
    return hashlib.md5(f"{ticker}_{start_date}".encode()).hexdigest()

def save_to_cache(key, data):
    cache_file = os.path.join(CACHE_DIR, f"{key}.pkl")
    with open(cache_file, 'wb') as f:
        pickle.dump({'data': data, 'timestamp': datetime.now()}, f)
    logger.debug(f"Data saved to cache: {cache_file}")

def load_from_cache(key, max_age_hours=24):
    cache_file = os.path.join(CACHE_DIR, f"{key}.pkl")
    if os.path.exists(cache_file):
        with open(cache_file, 'rb') as f:
            cached = pickle.load(f)
            if datetime.now() - cached['timestamp'] < timedelta(hours=max_age_hours):
                logger.debug(f"Cache hit: {cache_file}")
                return cached['data']
            else:
                logger.debug(f"Cache expired: {cache_file}")
    logger.debug(f"Cache miss: {cache_file}")
    return None

# Utility functions
@lru_cache(maxsize=100)
def get_ticker_info(ticker):
    try:
        stock = yf.Ticker(ticker)
        info = stock.info
        return {
            'sector': info.get('sector', 'Unknown'),
            'industry': info.get('industry', 'Unknown'),
            'name': info.get('longName', ticker)
        }
    except Exception as e:
        logger.error(f"Failed to fetch ticker info for {ticker}: {str(e)}")
        return {'sector': 'Unknown', 'industry': 'Unknown', 'name': ticker}

def validate_ticker(ticker):
    try:
        ticker = ticker.strip().upper()
        if not ticker:
            return False
        stock = yf.Ticker(ticker)
        info = stock.info
        return bool(info)
    except Exception as e:
        logger.error(f"Ticker validation failed for {ticker}: {str(e)}")
        return False

def fetch_stock_data(ticker, start_date):
    cache_key = get_cache_key(ticker, start_date)
    cached_data = load_from_cache(cache_key)

    if cached_data is not None:
        return cached_data

    try:
        stock_data = yf.download(ticker, start=start_date, progress=False)
        if not stock_data.empty:
            save_to_cache(cache_key, stock_data)
            logger.info(f"Data fetched and cached for {ticker} from {start_date}")
        else:
            logger.warning(f"No data fetched for {ticker} from {start_date}")
        return stock_data
    except Exception as e:
        logger.error(f"Failed to fetch data for {ticker}: {str(e)}")
        return pd.DataFrame()

def clean_for_json(obj):
    if isinstance(obj, (np.integer, np.floating)):
        return float(obj) if not (np.isnan(obj) or np.isinf(obj)) else None
    elif isinstance(obj, dict):
        return {k: clean_for_json(v) for k, v in obj.items()}
    elif isinstance(obj, (list, tuple)):
        return [clean_for_json(x) for x in obj]
    elif isinstance(obj, pd.Series):
        return [clean_for_json(x) for x in obj.values]
    elif isinstance(obj, pd.DataFrame):
        return {col: clean_for_json(obj[col]) for col in obj.columns}
    elif isinstance(obj, np.ndarray):
        return [clean_for_json(x) for x in obj.flatten()]
    elif pd.isna(obj):
        return None
    return obj

# Metric calculation functions
def calculate_portfolio_metrics(returns, benchmark_returns=None):
    """Calculate portfolio metrics with enhanced validation and debugging"""
    
    metrics = {}
    logger.info("Starting portfolio metrics calculation...")
    logger.info(f"Returns shape: {returns.shape}")
    logger.info(f"Returns head:\n{returns.head()}")
    logger.info(f"Returns tail:\n{returns.tail()}")
    
    try:
        # Calculate basic return metrics
        metrics['cagr'] = ep.annual_return(returns)
        logger.info(f"CAGR: {metrics['cagr']}")
        
        # Calculate total return
        total_return = (1 + returns).prod() - 1
        metrics['total_return'] = total_return
        logger.info(f"Total return: {total_return}")
        
        # Calculate positive periods
        total_periods = len(returns)
        positive_periods = (returns > 0).sum()
        metrics['positive_periods_pct'] = float(positive_periods) / total_periods
        logger.info(f"Positive periods: {positive_periods}/{total_periods} = {metrics['positive_periods_pct']}")
        
        # Calculate monthly returns
        monthly_returns = returns.resample('ME').agg(
            lambda x: (1 + x).prod() - 1
        )
        logger.info(f"Monthly returns sample:\n{monthly_returns.head()}")
        
        # Calculate yearly returns
        yearly_returns = monthly_returns.resample('YE').agg(
            lambda x: (1 + x).prod() - 1
        )
        logger.info(f"Yearly returns:\n{yearly_returns}")
        
        metrics['best_year'] = yearly_returns.max()
        metrics['worst_year'] = yearly_returns.min()
        logger.info(f"Best year: {metrics['best_year']}, Worst year: {metrics['worst_year']}")
        
        # Calculate risk metrics
        metrics['annual_volatility'] = ep.annual_volatility(returns)
        metrics['max_drawdown'] = ep.max_drawdown(returns)
        logger.info(f"Annual volatility: {metrics['annual_volatility']}")
        logger.info(f"Max drawdown: {metrics['max_drawdown']}")
        
        # Calculate VaR
        metrics['var_historical'] = monthly_returns.quantile(0.05)
        logger.info(f"VaR (5%): {metrics['var_historical']}")
        
        # Calculate gain/loss ratio
        monthly_gains = monthly_returns[monthly_returns > 0]
        monthly_losses = monthly_returns[monthly_returns < 0]
        if len(monthly_losses) > 0 and len(monthly_gains) > 0:
            metrics['gain_loss_ratio'] = abs(monthly_gains.mean() / monthly_losses.mean())
            logger.info(f"Gain/Loss ratio calculation:")
            logger.info(f"Average gains: {monthly_gains.mean()}")
            logger.info(f"Average losses: {monthly_losses.mean()}")
            logger.info(f"Gain/Loss ratio: {metrics['gain_loss_ratio']}")
        
        if benchmark_returns is not None:
            logger.info("Calculating benchmark-relative metrics...")
            
            # Align benchmark returns
            common_dates = returns.index.intersection(benchmark_returns.index)
            aligned_returns = returns[common_dates]
            aligned_benchmark = benchmark_returns[common_dates]
            
            # Calculate monthly returns for capture ratios
            monthly_portfolio = aligned_returns.resample('ME').agg(lambda x: (1 + x).prod() - 1)
            monthly_benchmark = aligned_benchmark.resample('ME').agg(lambda x: (1 + x).prod() - 1)
            
            # Calculate capture ratios
            up_months = monthly_benchmark > 0
            down_months = monthly_benchmark < 0
            
            logger.info("Up/Down market analysis:")
            logger.info(f"Total months: {len(monthly_benchmark)}")
            logger.info(f"Up months: {up_months.sum()}")
            logger.info(f"Down months: {down_months.sum()}")
            
            if up_months.any():
                up_portfolio = monthly_portfolio[up_months].mean()
                up_benchmark = monthly_benchmark[up_months].mean()
                metrics['upside_capture'] = up_portfolio / up_benchmark if up_benchmark != 0 else None
                logger.info(f"Upside capture calculation:")
                logger.info(f"Portfolio up return: {up_portfolio}")
                logger.info(f"Benchmark up return: {up_benchmark}")
                logger.info(f"Upside capture: {metrics['upside_capture']}")
            
            if down_months.any():
                down_portfolio = monthly_portfolio[down_months].mean()
                down_benchmark = monthly_benchmark[down_months].mean()
                metrics['downside_capture'] = down_portfolio / down_benchmark if down_benchmark != 0 else None
                logger.info(f"Downside capture calculation:")
                logger.info(f"Portfolio down return: {down_portfolio}")
                logger.info(f"Benchmark down return: {down_benchmark}")
                logger.info(f"Downside capture: {metrics['downside_capture']}")
            
            # Calculate other benchmark-relative metrics
            metrics['beta'] = ep.beta(aligned_returns, aligned_benchmark)
            metrics['alpha'] = ep.alpha(aligned_returns, aligned_benchmark, risk_free=0.02/252)
            metrics['correlation'] = aligned_returns.corr(aligned_benchmark)
            metrics['r_squared'] = metrics['correlation'] ** 2
            
            logger.info(f"Beta: {metrics['beta']}")
            logger.info(f"Alpha: {metrics['alpha']}")
            logger.info(f"Correlation: {metrics['correlation']}")
            logger.info(f"R-squared: {metrics['r_squared']}")
        
        # Calculate safe withdrawal rate
        metrics['safe_withdrawal_rate'] = 0.04 * (1 - abs(metrics['max_drawdown']))
        logger.info(f"Safe withdrawal rate: {metrics['safe_withdrawal_rate']}")
        
        # Clean up and validate metrics
        logger.info("Raw metrics before cleanup:")
        logger.info(metrics)
        
        for key, value in metrics.items():
            if value is None or np.isinf(value) or np.isnan(value):
                logger.warning(f"Invalid value for {key}: {value}")
                metrics[key] = None
            else:
                # Apply reasonable caps
                if key in ['upside_capture', 'downside_capture']:
                    original = value
                    metrics[key] = np.clip(value, -2, 2)
                    if metrics[key] != original:
                        logger.warning(f"Capped {key} from {original} to {metrics[key]}")
                elif key == 'positive_periods_pct':
                    metrics[key] = np.clip(value, 0, 1)
                elif key == 'gain_loss_ratio':
                    metrics[key] = np.clip(value, 0, 10)
                elif key in ['alpha', 'cagr', 'total_return', 'best_year', 'worst_year']:
                    original = value
                    metrics[key] = np.clip(value, -1, 2)
                    if metrics[key] != original:
                        logger.warning(f"Capped {key} from {original} to {metrics[key]}")
        
        logger.info("Final metrics after cleanup:")
        logger.info(metrics)
        
        return metrics
        
    except Exception as e:
        logger.error(f"Error in calculate_portfolio_metrics: {str(e)}")
        logger.error(f"Full traceback: {traceback.format_exc()}")
        return {}

def calculate_rebalancing_impact(portfolio_data, weights, transaction_cost=0.001):
    """Calculate the impact of different rebalancing strategies"""
    try:
        initial_investment = 10000
        results = {}
        
        # Convert weights to numpy array
        weights = np.array(weights)
        
        # Define rebalancing frequencies
        frequencies = {
            'null': None,  # Buy and hold
            'M': 'ME',     # Monthly
            'Q': 'QE',     # Quarterly
            'Y': 'YE'      # Yearly
        }

        for freq_key, freq in frequencies.items():
            portfolio_value = initial_investment
            positions = weights * initial_investment / portfolio_data.iloc[0]
            values = [initial_investment]
            total_cost = 0

            for i in range(1, len(portfolio_data)):
                # Update positions values
                current_prices = portfolio_data.iloc[i]
                current_values = positions * current_prices
                portfolio_value = current_values.sum()

                # Check if rebalancing is needed
                should_rebalance = False
                if freq:
                    current_date = portfolio_data.index[i]
                    if freq == 'ME' and current_date.is_month_end:
                        should_rebalance = True
                    elif freq == 'QE' and current_date.is_quarter_end:
                        should_rebalance = True
                    elif freq == 'YE' and current_date.is_year_end:
                        should_rebalance = True

                if should_rebalance:
                    # Calculate and apply rebalancing trades
                    target_values = portfolio_value * weights
                    trades = target_values - current_values
                    total_trades = np.abs(trades).sum()
                    cost = total_trades * transaction_cost
                    total_cost += cost
                    portfolio_value -= cost
                    positions = target_values / current_prices

                values.append(float(portfolio_value))

            # Calculate metrics
            returns = pd.Series(values).pct_change().dropna()
            total_return = values[-1] / values[0] - 1
            volatility = returns.std() * np.sqrt(252)
            sharpe = (returns.mean() * 252 - 0.02) / volatility if volatility != 0 else 0

            # Calculate drawdown
            cumulative = (1 + returns).cumprod()
            rolling_max = cumulative.expanding().max()
            drawdowns = cumulative / rolling_max - 1
            max_drawdown = drawdowns.min()

            results[freq_key] = {
                'final_value': float(values[-1]),
                'annualized_return': float((1 + total_return) ** (252/len(returns)) - 1),
                'volatility': float(volatility),
                'sharpe_ratio': float(sharpe),
                'max_drawdown': float(max_drawdown),
                'total_cost': float(total_cost),
                'cost_percentage': float(total_cost / initial_investment * 100),
                'values': values
            }

        return results

    except Exception as e:
        logger.error(f"Error in calculate_rebalancing_impact: {str(e)}")
        raise

def save_portfolio(portfolio_data):
    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
    filename = f"portfolio_{timestamp}.json"

    filepath = os.path.join(PORTFOLIO_DIR, filename)
    with open(filepath, 'w') as f:
        json.dump(portfolio_data, f)

    logger.info(f"Portfolio saved: {filename}")
    return filename

def load_portfolio(filename):
    filepath = os.path.join(PORTFOLIO_DIR, filename)
    try:
        with open(filepath, 'r') as f:
            portfolio = json.load(f)
        logger.info(f"Portfolio loaded: {filename}")
        return portfolio
    except Exception as e:
        logger.error(f"Failed to load portfolio {filename}: {str(e)}")
        return None

def generate_csv_report(results):
    output = io.StringIO()
    writer = csv.writer(output)

    writer.writerow(['Portfolio Metrics'])
    writer.writerow(['Metric', 'Portfolio', 'Benchmark'])
    for key in results.get('portfolio_metrics', {}):
        writer.writerow([
            key,
            results['portfolio_metrics'][key],
            results['benchmark_metrics'].get(key, 'N/A')
        ])

    writer.writerow([])
    writer.writerow(['Historical Performance'])
    writer.writerow(['Date', 'Portfolio Value', 'Benchmark Value'])
    for i, date in enumerate(results['historical_performance']['dates']):
        writer.writerow([
            date,
            results['historical_performance']['portfolio_values'][i],
            results['historical_performance']['benchmark_values'][i]
        ])

    csv_content = output.getvalue()
    output.close()
    logger.info("CSV report generated successfully")
    return csv_content

def get_sector_info(ticker):
    """Get sector information with better mapping and caching"""
    try:
        # Manual mapping for common ETFs
        etf_mapping = {
            'VOO': {'sector': 'ETF', 'industry': 'S&P 500 Index', 'name': 'Vanguard S&P 500 ETF'},
            'SPY': {'sector': 'ETF', 'industry': 'S&P 500 Index', 'name': 'SPDR S&P 500 ETF'},
            'QQQ': {'sector': 'ETF', 'industry': 'NASDAQ Index', 'name': 'Invesco QQQ Trust'},
            'IWM': {'sector': 'ETF', 'industry': 'Russell 2000 Index', 'name': 'iShares Russell 2000 ETF'},
            # Add more ETFs as needed
        }
        
        # Check if it's a known ETF
        if ticker in etf_mapping:
            return etf_mapping[ticker]
            
        # Get info from yfinance
        stock = yf.Ticker(ticker)
        info = stock.info
        
        # Better sector mapping
        sector = info.get('sector')
        if not sector or sector == 'Unknown':
            # Try to determine sector from industry
            industry = info.get('industry', '').lower()
            if 'technology' in industry:
                sector = 'Technology'
            elif 'financial' in industry:
                sector = 'Financial'
            elif 'healthcare' in industry:
                sector = 'Healthcare'
            elif 'consumer' in industry:
                sector = 'Consumer' + (' Cyclical' if 'cyclical' in industry else ' Defensive')
            else:
                sector = 'Other'
        
        return {
            'sector': sector,
            'industry': info.get('industry', 'Unknown'),
            'name': info.get('longName', ticker)
        }
    except Exception as e:
        logger.error(f"Failed to fetch ticker info for {ticker}: {str(e)}")
        return {'sector': 'Other', 'industry': 'Unknown', 'name': ticker}

def calculate_attribution(portfolio_data, weights, benchmark_data):
    try:
        logger.info("Starting attribution calculation...")
        logger.debug(f"Portfolio data shape: {portfolio_data.shape}")
        logger.debug(f"Weights length: {len(weights)}")
        logger.debug(f"Benchmark data shape: {benchmark_data.shape}")

        # Align dates
        common_dates = portfolio_data.index.intersection(benchmark_data.index)
        portfolio_data = portfolio_data.reindex(common_dates)
        benchmark_data = benchmark_data.reindex(common_dates)

        # Calculate returns
        portfolio_returns = portfolio_data.pct_change().fillna(0)
        benchmark_returns = benchmark_data.pct_change().fillna(0)

        # Calculate weighted returns
        weighted_returns = portfolio_returns.multiply(weights, axis=1)

        # Get sector information for each ticker
        sectors = {ticker: get_sector_info(ticker) for ticker in portfolio_data.columns}
        
        # Group by sectors
        sector_attribution = {}
        for ticker, info in sectors.items():
            sector = info['sector']
            weight = weights[list(portfolio_data.columns).index(ticker)]
            returns = weighted_returns[ticker]
            annual_return = (1 + returns).prod() ** (252 / len(returns)) - 1
            
            if sector not in sector_attribution:
                sector_attribution[sector] = {
                    'weight': 0,
                    'return': 0,
                    'contribution': 0
                }
            
            sector_attribution[sector]['weight'] += weight
            # Weight the return by the proportion within the sector
            sector_attribution[sector]['return'] = (
                (sector_attribution[sector]['return'] * 
                 (sector_attribution[sector]['weight'] - weight) +
                 annual_return * weight) / 
                sector_attribution[sector]['weight']
            )
            sector_attribution[sector]['contribution'] += annual_return * weight

        # Calculate total return
        total_return = sum(data['contribution'] for data in sector_attribution.values())

        # Factor Attribution
        portfolio_cumulative = (1 + weighted_returns.sum(axis=1)).cumprod()
        benchmark_cumulative = (1 + benchmark_returns).cumprod()

        # Calculate annualized returns
        years = (common_dates[-1] - common_dates[0]).days / 365.25
        portfolio_return = (portfolio_cumulative.iloc[-1]) ** (1 / years) - 1
        benchmark_return = (benchmark_cumulative.iloc[-1]) ** (1 / years) - 1

        # Ensure both series are one-dimensional and have the same length
        benchmark_returns_aligned = benchmark_returns.loc[common_dates].values.flatten()
        portfolio_returns_aligned = weighted_returns.sum(axis=1).loc[common_dates].values.flatten()

        # Calculate beta and alpha using regression
        model = stats.linregress(benchmark_returns_aligned, portfolio_returns_aligned)
        beta = model.slope
        alpha = model.intercept * 252  # Annualize alpha
        r_squared = model.rvalue ** 2

        factor_attribution = {
            'market_beta': float(beta),
            'alpha': float(alpha),
            'r_squared': float(r_squared),
            'portfolio_return': float(portfolio_return),
            'benchmark_return': float(benchmark_return.iloc[0]) if isinstance(benchmark_return, pd.Series) else float(benchmark_return)
        }

        attribution = {
            'sector': sector_attribution,
            'factor': factor_attribution,
            'total_return': float(total_return)
        }

        logger.info("Attribution calculation completed:")
        logger.debug(f"Sector attribution: {sector_attribution}")
        logger.debug(f"Factor attribution: {factor_attribution}")
        logger.debug(f"Total return: {total_return}")

        # Calculate relative contribution percentages
        total_abs_contribution = sum(abs(data['contribution']) for data in sector_attribution.values())
        for sector in sector_attribution:
            sector_attribution[sector]['relative_contribution'] = (
                abs(sector_attribution[sector]['contribution']) / total_abs_contribution
                if total_abs_contribution != 0 else 0
            )

        return clean_for_json(attribution)

    except Exception as e:
        logger.error(f"Error in calculate_attribution: {str(e)}", exc_info=True)
        raise

def calculate_correlation_matrix(portfolio_data):
    """Calculate correlation matrix with proper handling of returns"""
    try:
        # Calculate returns first
        returns = portfolio_data.pct_change().dropna()
        
        # Calculate correlations using returns, not prices
        pearson_corr = returns.corr(method='pearson')
        spearman_corr = returns.corr(method='spearman')

        # Create mask for diagonal elements
        mask = ~np.eye(len(pearson_corr), dtype=bool)

        # Calculate statistics excluding self-correlations
        pearson_stats = {
            'min': float(np.min(pearson_corr.values[mask])),
            'max': float(np.max(pearson_corr.values[mask])),
            'mean': float(np.mean(pearson_corr.values[mask]))
        }

        spearman_stats = {
            'min': float(np.min(spearman_corr.values[mask])),
            'max': float(np.max(spearman_corr.values[mask])),
            'mean': float(np.mean(spearman_corr.values[mask]))
        }

        correlations = {
            'pearson': {
                'matrix': pearson_corr.to_dict(),
                'min': pearson_stats['min'],
                'max': pearson_stats['max'],
                'mean': pearson_stats['mean']
            },
            'spearman': {
                'matrix': spearman_corr.to_dict(),
                'min': spearman_stats['min'],
                'max': spearman_stats['max'],
                'mean': spearman_stats['mean']
            }
        }

        logger.info("Correlation calculation completed:")
        logger.debug(f"Pearson stats: {pearson_stats}")
        logger.debug(f"Spearman stats: {spearman_stats}")

        return clean_for_json(correlations)

    except Exception as e:
        logger.error(f"Error in calculate_correlation_matrix: {str(e)}", exc_info=True)
        raise

@app.route('/api/compare', methods=['POST'])
def compare_portfolios():
    try:
        logger.info("Starting portfolio comparison...")
        data = request.json
        logger.info(f"Received request data: {json.dumps(data, indent=2)}")
        
        portfolios = data.get('portfolios', [])
        start_date = data.get('startDate')
        transaction_cost = data.get('transactionCost', 0.001)

        if not portfolios:
            return jsonify({
                'success': False,
                'error': 'No portfolios provided for comparison'
            }), 400

        start_date = pd.to_datetime(start_date)
        all_data = pd.DataFrame()
        comparison_results = {}
        
        # Collect all unique tickers
        unique_tickers = set()
        for portfolio in portfolios:
            portfolio_stocks = portfolio.get('portfolio', [])
            unique_tickers.update([stock['ticker'] for stock in portfolio_stocks])

        # Fetch data for all tickers
        logger.info(f"Fetching data for {len(unique_tickers)} unique tickers...")
        for ticker in unique_tickers:
            stock_data = fetch_stock_data(ticker, start_date)
            if stock_data is not None and not stock_data.empty:
                all_data[ticker] = stock_data['Adj Close']
            else:
                return jsonify({
                    'success': False,
                    'error': f'No data available for {ticker}'
                }), 400

        # First pass: Calculate and store returns
        stored_returns = {}
        for portfolio_data in portfolios:
            portfolio_name = portfolio_data.get('name', 'Unnamed Portfolio')
            portfolio_stocks = portfolio_data.get('portfolio', [])
            
            tickers = [stock['ticker'] for stock in portfolio_stocks]
            weights = [float(stock['weight']) for stock in portfolio_stocks]
            
            portfolio_price_data = all_data[tickers].dropna()
            portfolio_returns = portfolio_price_data.pct_change().fillna(0)
            weighted_returns = portfolio_returns.dot(weights)
            stored_returns[portfolio_name] = weighted_returns

        # Second pass: Calculate metrics using stored returns
        for portfolio_name, weighted_returns in stored_returns.items():
            portfolio_cumulative = (1 + weighted_returns).cumprod()

            # Calculate total return and basic metrics
            total_return = portfolio_cumulative.iloc[-1] - 1
            years = (portfolio_cumulative.index[-1] - portfolio_cumulative.index[0]).days / 365.25
            cagr = (1 + total_return) ** (1/years) - 1 if years > 0 else 0

            # Calculate monthly and yearly returns
            monthly_returns = weighted_returns.resample('M').apply(
                lambda x: (1 + x).prod() - 1
            )
            yearly_returns = monthly_returns.resample('Y').apply(
                lambda x: (1 + x).prod() - 1
            )

            # Calculate drawdowns
            rolling_max = portfolio_cumulative.expanding().max()
            drawdown_series = portfolio_cumulative / rolling_max - 1
            max_drawdown = drawdown_series.min()

            # Calculate arithmetic and geometric means
            arithmetic_mean_monthly = monthly_returns.mean()
            arithmetic_mean_annual = (1 + arithmetic_mean_monthly)**12 - 1
            geometric_mean_monthly = (1 + monthly_returns).prod() ** (1/len(monthly_returns)) - 1
            geometric_mean_annual = (1 + geometric_mean_monthly)**12 - 1

            # Calculate risk metrics
            annual_std = weighted_returns.std() * np.sqrt(252)
            monthly_std = monthly_returns.std() * np.sqrt(12)
            downside_returns = monthly_returns[monthly_returns < 0]
            downside_std = np.sqrt(np.mean(downside_returns**2)) if len(downside_returns) > 0 else 0
            var_historical = np.percentile(monthly_returns, 5)

            # Calculate positive periods correctly
            positive_periods = (weighted_returns > 0).sum()
            total_periods = len(weighted_returns)
            positive_periods_pct = (positive_periods / total_periods) * 100  # Convert to percentage, but keep it reasonable

            # Calculate gain/loss ratio
            gains = monthly_returns[monthly_returns > 0]
            losses = monthly_returns[monthly_returns < 0]
            gain_loss_ratio = abs(gains.mean()) / abs(losses.mean()) if len(losses) > 0 else float('inf')

            # Calculate safe withdrawal rate (4% rule adjusted by drawdown)
            safe_withdrawal_rate = 4.0 * (1 + max_drawdown)  # Adjust 4% rule by drawdown

            # Initialize metrics dictionary
            metrics = {
                'start_balance': 10000,
                'end_balance': float(10000 * (1 + total_return)),
                'cagr': float(cagr),
                'best_year': float(yearly_returns.max()),
                'worst_year': float(yearly_returns.min()),
                'max_drawdown': float(max_drawdown),
                'sharpe_ratio': float(ep.sharpe_ratio(weighted_returns, risk_free=0.02/252)),
                'sortino_ratio': float(ep.sortino_ratio(weighted_returns, required_return=0.02/252)),
                'arithmetic_mean_annual': float(arithmetic_mean_annual),
                'geometric_mean_annual': float(geometric_mean_annual),
                'annual_volatility': float(annual_std),
                'monthly_volatility': float(monthly_std),
                'downside_std': float(downside_std),
                'var_historical': float(var_historical),
                'positive_periods_pct': float(positive_periods_pct),
                'gain_loss_ratio': float(gain_loss_ratio),
                'safe_withdrawal_rate': float(safe_withdrawal_rate)
            }

            # Calculate benchmark-relative metrics
            if portfolio_name == portfolios[0]['name']:  # For main portfolio
                benchmark_returns = stored_returns[portfolios[1]['name']]
                
                correlation = weighted_returns.corr(benchmark_returns)
                beta = correlation * (weighted_returns.std() / benchmark_returns.std())
                
                benchmark_cumulative = (1 + benchmark_returns).cumprod()
                benchmark_total_return = benchmark_cumulative.iloc[-1] - 1
                benchmark_cagr = (1 + benchmark_total_return) ** (1/years) - 1
                
                risk_free_rate = 0.02
                alpha = cagr - (risk_free_rate + beta * (benchmark_cagr - risk_free_rate))

                # Calculate upside/downside capture properly
                up_market = benchmark_returns > 0
                down_market = benchmark_returns < 0
                
                if any(up_market):
                    upside_returns = weighted_returns[up_market].mean()
                    benchmark_up_returns = benchmark_returns[up_market].mean()
                    upside_capture = (upside_returns / benchmark_up_returns) * 100  # Convert to percentage
                else:
                    upside_capture = 0
                    
                if any(down_market):
                    downside_returns = weighted_returns[down_market].mean()
                    benchmark_down_returns = benchmark_returns[down_market].mean()
                    downside_capture = (downside_returns / benchmark_down_returns) * 100  # Convert to percentage
                else:
                    downside_capture = 0

                # Calculate safe withdrawal rate properly
                safe_withdrawal_rate = 4.0 * (1 + max_drawdown)  # Base it on the 4% rule and drawdown

                metrics.update({
                    'correlation': float(correlation),
                    'beta': float(beta),
                    'alpha': float(alpha),
                    'upside_capture': float(upside_capture),
                    'downside_capture': float(downside_capture),
                    'r_squared': float(correlation ** 2),
                    'positive_periods_pct': float(positive_periods_pct),
                    'safe_withdrawal_rate': float(safe_withdrawal_rate)
                })
            else:
                # For benchmark portfolio
                metrics.update({
                    'correlation': None,
                    'beta': None,
                    'alpha': None,
                    'upside_capture': None,
                    'downside_capture': None,
                    'r_squared': None
                })

            # Store results
            comparison_results[portfolio_name] = {
                'metrics': metrics,
                'returns': weighted_returns.tolist(),
                'cumulative_returns': portfolio_cumulative.tolist(),
                'drawdown_series': drawdown_series.tolist(),
            }

        # Calculate rebalancing impact for the main portfolio
        main_portfolio = portfolios[0]
        main_portfolio_stocks = main_portfolio.get('portfolio', [])
        main_tickers = [stock['ticker'] for stock in main_portfolio_stocks]
        main_weights = [float(stock['weight']) for stock in main_portfolio_stocks]
        
        main_portfolio_data = all_data[main_tickers].dropna()
        rebalancing_results = calculate_rebalancing_impact(
            portfolio_data=main_portfolio_data,
            weights=main_weights,
            transaction_cost=transaction_cost
        )

        # Add rebalancing results to the response
        comparison_results['Current Portfolio']['rebalancing'] = rebalancing_results

        # Ensure we have dates before returning
        dates = all_data.index.strftime('%Y-%m-%d').tolist()
        if not dates:
            logger.error("No dates available in the response")
            return jsonify({
                'success': False,
                'error': 'No dates available for the selected time period'
            }), 400

        logger.info(f"Returning {len(dates)} dates")
        logger.info(f"First date: {dates[0]}, Last date: {dates[-1]}")
        logger.info(f"Number of portfolios: {len(comparison_results)}")

        response_data = {
            'success': True,
            'comparison_results': comparison_results,
            'dates': dates
        }

        logger.info("Response prepared successfully")
        return jsonify(response_data)

    except Exception as e:
        logger.error(f"Error in compare_portfolios: {str(e)}", exc_info=True)
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

# Add this function to calculate yearly returns
def calculate_yearly_metrics(returns):
    yearly_returns = returns.resample('Y').apply(
        lambda x: (1 + x).prod() - 1
    )
    return {
        'best_year': float(yearly_returns.max()),
        'worst_year': float(yearly_returns.min())
    }

# API Endpoints

@app.route('/api/validate-ticker', methods=['POST'])
def validate_ticker_endpoint():
    data = request.json
    ticker = data.get('ticker', '')
    is_valid = validate_ticker(ticker)
    logger.info(f"Ticker validation for '{ticker}': {is_valid}")
    return jsonify({
        'success': True,
        'valid': is_valid
    })

@app.route('/api/backtest', methods=['POST'])
def backtest():
    try:
        logger.info("=== Starting Backtest ===")
        data = request.json
        logger.info(f"Request data: {data}")

        portfolio = data['portfolio']
        start_date = data.get('startDate')
        benchmark = data.get('benchmark', 'SPY')

        logger.info(f"Processing portfolio: {portfolio}")
        logger.info(f"Start date: {start_date}")
        logger.info(f"Benchmark: {benchmark}")

        # Get portfolio data
        portfolio_data = pd.DataFrame()
        start_date = pd.to_datetime(start_date)

        # Fetch data using fetch_stock_data and calculate weights
        stock_data_dict = {}
        weights = []
        for stock in portfolio:
            ticker = stock['ticker'].strip().upper()
            weight = float(stock['weight']) / 100

            stock_data = fetch_stock_data(ticker, start_date)
            if stock_data is None or stock_data.empty:
                logger.error(f"No data available for {ticker}")
                return jsonify({
                    'success': False,
                    'error': f'No data available for {ticker}'
                }), 400

            stock_data_dict[ticker] = stock_data['Adj Close']
            weights.append(weight)

        # Validate that weights sum to 1
        total_weight = sum(weights)
        if not np.isclose(total_weight, 1.0):
            logger.error(f"Portfolio weights sum to {total_weight * 100:.2f}%, which is not 100%.")
            return jsonify({
                'success': False,
                'error': f'Portfolio weights must sum to 100%. Current sum: {total_weight * 100:.2f}%'
            }), 400

        # Fetch benchmark data
        benchmark_data = fetch_stock_data(benchmark, start_date)
        if benchmark_data is None or benchmark_data.empty:
            logger.error(f"No data available for benchmark {benchmark}")
            return jsonify({
                'success': False,
                'error': f'No data available for benchmark {benchmark}'
            }), 400
        benchmark_data = benchmark_data['Adj Close']

        # Create aligned portfolio data
        common_dates = None
        for ticker, data in stock_data_dict.items():
            if common_dates is None:
                common_dates = data.index
            else:
                common_dates = common_dates.intersection(data.index)

        # Add benchmark dates to intersection
        common_dates = common_dates.intersection(benchmark_data.index)

        # Align all data to common dates
        for ticker in stock_data_dict:
            portfolio_data[ticker] = stock_data_dict[ticker].reindex(common_dates)
        benchmark_data = benchmark_data.reindex(common_dates)

        # Drop any remaining NaNs
        portfolio_data = portfolio_data.dropna()
        benchmark_data = benchmark_data.dropna()

        if portfolio_data.empty or benchmark_data.empty:
            logger.error("Insufficient overlapping data between portfolio and benchmark.")
            return jsonify({
                'success': False,
                'error': 'Insufficient overlapping data between portfolio and benchmark.'
            }), 400

        # Calculate returns
        portfolio_returns = portfolio_data.pct_change().fillna(0)
        benchmark_returns = benchmark_data.pct_change().fillna(0)  # Already a Series

        # Calculate metrics
        portfolio_metrics = calculate_portfolio_metrics(
            portfolio_data=portfolio_data,
            weights=weights,
            benchmark_returns=benchmark_returns
        )

        # Create benchmark DataFrame with proper dimensions
        benchmark_df = pd.DataFrame()
        benchmark_df[benchmark] = benchmark_data

        benchmark_metrics = calculate_portfolio_metrics(
            portfolio_data=benchmark_df,
            weights=[1.0],
            benchmark_returns=None
        )

        # Calculate historical performance
        initial_investment = 10000
        portfolio_values = initial_investment * (1 + portfolio_returns.dot(weights)).cumprod()
        benchmark_values = initial_investment * (1 + benchmark_returns).cumprod()

        # Ensure both series have the same index
        common_dates = portfolio_values.index.intersection(benchmark_values.index)
        portfolio_values = portfolio_values.reindex(common_dates)
        benchmark_values = benchmark_values.reindex(common_dates)

        # Calculate drawdown periods for both portfolio and benchmark
        portfolio_drawdowns = calculate_drawdown_periods(
            portfolio_returns.dot(weights),
            portfolio_data.index
        )
        
        benchmark_drawdowns = calculate_drawdown_periods(
            benchmark_returns,
            benchmark_data.index
        )
        
        # Log drawdown calculations
        logger.info(f"Portfolio drawdowns calculated: {len(portfolio_drawdowns)} periods")
        logger.info(f"Benchmark drawdowns calculated: {len(benchmark_drawdowns)} periods")
        
        response_data = {
            'success': True,
            'portfolio_metrics': clean_for_json(portfolio_metrics),
            'benchmark_metrics': clean_for_json(benchmark_metrics),
            'historical_performance': {
                'dates': common_dates.strftime('%Y-%m-%d').tolist(),
                'portfolio_values': clean_for_json(portfolio_values.values),
                'benchmark_values': clean_for_json(benchmark_values.values),
                'initial_investment': initial_investment
            },
            'drawdowns': {
                'portfolio_drawdowns': portfolio_drawdowns,
                'benchmark_drawdowns': benchmark_drawdowns
            }
        }

        logger.info("Response data prepared successfully")
        return jsonify(response_data)

    except Exception as e:
        logger.error(f"Error in backtest: {str(e)}", exc_info=True)
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

@app.route('/api/portfolio/save', methods=['POST'])
def save_portfolio_endpoint():
    try:
        portfolio_data = request.json
        filename = save_portfolio(portfolio_data)
        return jsonify({
            'success': True,
            'filename': filename
        })
    except Exception as e:
        logger.error(f"Error in save_portfolio_endpoint: {str(e)}", exc_info=True)
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

@app.route('/api/portfolio/load', methods=['GET'])
def list_portfolios():
    try:
        portfolios = []
        for filename in os.listdir(PORTFOLIO_DIR):
            if filename.endswith('.json'):
                portfolio = load_portfolio(filename)
                if portfolio:
                    portfolios.append({
                        'filename': filename,
                        'portfolio': portfolio
                    })
        logger.info("Portfolios listed successfully")
        return jsonify({
            'success': True,
            'portfolios': portfolios
        })
    except Exception as e:
        logger.error(f"Error in list_portfolios: {str(e)}", exc_info=True)
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

@app.route('/api/export/csv', methods=['POST'])
def export_csv():
    try:
        results = request.json
        csv_data = generate_csv_report(results)
        logger.info("CSV export successful")
        return jsonify({
            'success': True,
            'csv_data': csv_data
        })
    except Exception as e:
        logger.error(f"Error in export_csv: {str(e)}", exc_info=True)
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

@app.route('/api/attribution', methods=['POST'])
def get_attribution():
    try:
        logger.info("Starting attribution analysis...")
        data = request.json
        logger.info(f"Received data: {json.dumps(data, indent=2)}")
        
        portfolio = data.get('portfolio', [])
        start_date = data.get('startDate')
        benchmark = data.get('benchmark', 'SPY')

        if not isinstance(portfolio, list):
            logger.error(f"Portfolio is not a list: {type(portfolio)}")
            return jsonify({
                'success': False,
                'error': 'Portfolio must be a list of stocks'
            }), 400

        if not portfolio:
            return jsonify({
                'success': False,
                'error': 'No portfolio provided'
            }), 400

        start_date = pd.to_datetime(start_date)
        all_data = pd.DataFrame()

        # Fetch data for portfolio stocks
        for stock in portfolio:
            ticker = stock['ticker']
            weight = float(stock['weight'])
            stock_data = fetch_stock_data(ticker, start_date)
            if stock_data is not None and not stock_data.empty:
                all_data[ticker] = stock_data['Adj Close']
            else:
                return jsonify({
                    'success': False,
                    'error': f'No data available for {ticker}'
                }), 400

        # Fetch benchmark data
        logger.info("Fetching benchmark data...")
        benchmark_data = fetch_stock_data(benchmark, start_date)
        if benchmark_data is None or benchmark_data.empty:
            return jsonify({
                'success': False,
                'error': f'No data available for benchmark {benchmark}'
            }), 400

        benchmark_prices = benchmark_data['Adj Close']

        # Find common dates and align data
        logger.info("Finding common dates...")
        common_dates = all_data.index.intersection(benchmark_prices.index)
        if len(common_dates) == 0:
            return jsonify({
                'success': False,
                'error': 'No overlapping dates found'
            }), 400

        # Align data to common dates
        portfolio_data = all_data.reindex(common_dates)
        benchmark_prices = benchmark_prices.reindex(common_dates)

        # Add debug logging for data shapes
        logger.info(f"Portfolio data shape: {portfolio_data.shape}")
        logger.info(f"Benchmark prices shape: {benchmark_prices.shape}")

        # Calculate returns and ensure they're Series
        portfolio_returns = portfolio_data.pct_change().fillna(0)
        benchmark_returns = benchmark_prices.pct_change().fillna(0)

        # Calculate weighted returns
        weights = np.array([float(stock['weight']) for stock in portfolio])
        weighted_returns = portfolio_returns.dot(weights)

        # Convert benchmark returns to Series if it's a DataFrame
        if isinstance(benchmark_returns, pd.DataFrame):
            benchmark_returns = benchmark_returns.iloc[:, 0]

        # Calculate factor metrics with proper annualization
        correlation = weighted_returns.corr(benchmark_returns)
        beta = correlation * (weighted_returns.std() / benchmark_returns.std())
        
        # Calculate returns and annualize them
        portfolio_total_return = (1 + weighted_returns).prod() - 1
        benchmark_total_return = (1 + benchmark_returns).prod() - 1
        
        # Calculate years
        years = (portfolio_data.index[-1] - portfolio_data.index[0]).days / 365.25
        
        # Calculate annualized returns
        portfolio_cagr = (1 + portfolio_total_return) ** (1/years) - 1
        benchmark_cagr = (1 + benchmark_total_return) ** (1/years) - 1
        
        # Calculate alpha using CAPM with annualized returns
        risk_free_rate = 0.02  # 2% annual risk-free rate
        alpha = portfolio_cagr - (risk_free_rate + beta * (benchmark_cagr - risk_free_rate))
        
        # Calculate R-squared
        r_squared = correlation ** 2

        # Initialize attribution dictionary with all required keys
        attribution = {
            'sector': {},
            'stock': {},  # Initialize stock dictionary
            'timing': {},
            'factor': {}
        }

        # Store factor attribution with proper metrics
        attribution['factor'] = {
            'market_beta': float(beta),
            'alpha': float(alpha),  # Already annualized
            'r_squared': float(r_squared),
            'portfolio_return': float(portfolio_cagr),  # Use CAGR
            'benchmark_return': float(benchmark_cagr)  # Use CAGR
        }

        # Calculate stock attribution
        for ticker in portfolio_data.columns:
            stock_returns = portfolio_returns[ticker]
            weight = float(next(s['weight'] for s in portfolio if s['ticker'] == ticker))
            stock_info = get_ticker_info(ticker)
            sector = stock_info['sector']
            
            # Calculate contribution
            contribution = stock_returns * weight
            total_contribution = float((1 + contribution).prod() - 1)
            
            # Store stock attribution
            attribution['stock'][ticker] = {
                'weight': weight,
                'return': float((1 + stock_returns).prod() - 1),
                'contribution': total_contribution,
                'sector': sector
            }
            
            # Initialize sector if not exists
            if sector not in attribution['sector']:
                attribution['sector'][sector] = {
                    'weight': 0,
                    'return': 0,
                    'contribution': 0
                }
            
            # Update sector data
            attribution['sector'][sector]['weight'] += weight
            attribution['sector'][sector]['contribution'] += total_contribution

        # Calculate sector returns and relative contributions
        total_contribution = sum(s['contribution'] for s in attribution['sector'].values())
        for sector in attribution['sector']:
            sector_data = attribution['sector'][sector]
            if sector_data['weight'] > 0:
                sector_data['return'] = sector_data['contribution'] / sector_data['weight']
                sector_data['relative_contribution'] = (
                    sector_data['contribution'] / total_contribution if total_contribution != 0 else 0
                )

        # Calculate timing effect
        attribution['timing'] = {
            'portfolio_return': float(portfolio_cagr),
            'benchmark_return': float(benchmark_cagr),
            'excess_return': float(portfolio_cagr - benchmark_cagr)
        }

        logger.info("Attribution calculation completed successfully")
        logger.info(f"Factor metrics: {attribution['factor']}")

        return jsonify({
            'success': True,
            'attribution': attribution,
            'dates': portfolio_data.index.strftime('%Y-%m-%d').tolist()
        })

    except Exception as e:
        logger.error(f"Error in get_attribution: {str(e)}", exc_info=True)
        logger.error(f"Full traceback: {traceback.format_exc()}")
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

@app.route('/api/rebalance', methods=['POST'])
def analyze_rebalancing():
    try:
        data = request.json
        portfolio = data['portfolio']
        start_date = data.get('startDate')
        start_date = pd.to_datetime(start_date)

        portfolio_data = pd.DataFrame()
        weights = []

        for stock in portfolio:
            ticker = stock['ticker'].strip().upper()
            weight = float(stock['weight']) / 100
            stock_data = fetch_stock_data(ticker, start_date)
            if stock_data is None or stock_data.empty:
                logger.error(f"No data available for {ticker}")
                return jsonify({
                    'success': False,
                    'error': f'No data available for {ticker}'
                }), 400
            portfolio_data[ticker] = stock_data['Adj Close']
            weights.append(weight)

        # Validate that weights sum to 1
        total_weight = sum(weights)
        if not np.isclose(total_weight, 1.0):
            logger.error(f"Portfolio weights sum to {total_weight * 100:.2f}%, which is not 100%.")
            return jsonify({
                'success': False,
                'error': f'Portfolio weights must sum to 100%. Current sum: {total_weight * 100:.2f}%'
            }), 400

        weights = np.array(weights)

        rebalancing_results = calculate_rebalancing_impact(portfolio_data, weights)

        return jsonify({
            'success': True,
            'rebalancing_results': rebalancing_results
        })

    except Exception as e:
        logger.error(f"Error in analyze_rebalancing: {str(e)}", exc_info=True)
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

@app.route('/api/cache/clear', methods=['POST'])
def clear_cache():
    try:
        for file in os.listdir(CACHE_DIR):
            if file.endswith('.pkl'):
                os.remove(os.path.join(CACHE_DIR, file))
                logger.debug(f"Cache file removed: {file}")
        logger.info("Cache cleared successfully")
        return jsonify({
            'success': True,
            'message': 'Cache cleared successfully'
        })
    except Exception as e:
        logger.error(f"Error in clear_cache: {str(e)}", exc_info=True)
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

@app.route('/api/cache/status', methods=['GET'])
def cache_status():
    try:
        cache_files = [f for f in os.listdir(CACHE_DIR) if f.endswith('.pkl')]
        total_size = sum(os.path.getsize(os.path.join(CACHE_DIR, f)) for f in cache_files)

        logger.info("Cache status retrieved successfully")
        return jsonify({
            'success': True,
            'status': {
                'cached_items': len(cache_files),
                'total_size_mb': round(total_size / (1024 * 1024), 2),
                'cache_dir': CACHE_DIR
            }
        })
    except Exception as e:
        logger.error(f"Error in cache_status: {str(e)}", exc_info=True)
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

@app.route('/api/portfolio/delete', methods=['POST'])
def delete_portfolio():
    try:
        data = request.json
        filename = data.get('filename')
        
        if not filename:
            return jsonify({
                'success': False,
                'error': 'No filename provided'
            }), 400
            
        filepath = os.path.join(PORTFOLIO_DIR, filename)
        
        # Check if file exists
        if not os.path.exists(filepath):
            return jsonify({
                'success': False,
                'error': 'Portfolio not found'
            }), 404
            
        # Delete the file
        os.remove(filepath)
        logger.info(f"Portfolio deleted: {filename}")
        
        # Return updated list of portfolios
        portfolios = []
        for remaining_file in os.listdir(PORTFOLIO_DIR):
            if remaining_file.endswith('.json'):
                portfolio = load_portfolio(remaining_file)
                if portfolio:
                    portfolios.append({
                        'filename': remaining_file,
                        'portfolio': portfolio
                    })
        
        return jsonify({
            'success': True,
            'message': 'Portfolio deleted successfully',
            'portfolios': portfolios
        })
        
    except Exception as e:
        logger.error(f"Error in delete_portfolio: {str(e)}", exc_info=True)
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

@app.route('/api/portfolio/update-name', methods=['POST'])
def update_portfolio_name():
    try:
        data = request.json
        filename = data.get('filename')
        new_name = data.get('name')
        
        if not filename or not new_name:
            return jsonify({
                'success': False,
                'error': 'Filename and new name are required'
            }), 400
            
        filepath = os.path.join(PORTFOLIO_DIR, filename)
        
        # Check if file exists
        if not os.path.exists(filepath):
            return jsonify({
                'success': False,
                'error': 'Portfolio not found'
            }), 404
            
        # Load existing portfolio
        with open(filepath, 'r') as f:
            portfolio_data = json.load(f)
            
        # Update name
        portfolio_data['name'] = new_name
        
        # Save updated portfolio
        with open(filepath, 'w') as f:
            json.dump(portfolio_data, f)
            
        logger.info(f"Portfolio name updated: {filename} -> {new_name}")
        
        # Return updated list of portfolios
        portfolios = []
        for file in os.listdir(PORTFOLIO_DIR):
            if file.endswith('.json'):
                portfolio = load_portfolio(file)
                if portfolio:
                    portfolios.append({
                        'filename': file,
                        'portfolio': portfolio
                    })
        
        return jsonify({
            'success': True,
            'message': 'Portfolio name updated successfully',
            'portfolios': portfolios
        })
        
    except Exception as e:
        logger.error(f"Error in update_portfolio_name: {str(e)}", exc_info=True)
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

@app.route('/api/health', methods=['GET'])
def health_check():
    return jsonify({
        'success': True,
        'message': 'Backend is running',
        'timestamp': datetime.now().isoformat()
    })

@app.route('/api/correlation', methods=['POST'])
def get_correlation():
    try:
        data = request.json
        portfolio = data.get('portfolio', [])
        start_date = data.get('startDate')
        start_date = pd.to_datetime(start_date)

        # Prepare portfolio data
        portfolio_data = pd.DataFrame()
        for stock in portfolio:
            ticker = stock['ticker'].strip().upper()
            stock_data = fetch_stock_data(ticker, start_date)
            if stock_data is None or stock_data.empty:
                logger.error(f"No data available for {ticker}")
                return jsonify({
                    'success': False,
                    'error': f'No data available for {ticker}'
                }), 400
            portfolio_data[ticker] = stock_data['Adj Close']

        if portfolio_data.empty:
            logger.error("No data available for correlation calculation")
            return jsonify({
                'success': False,
                'error': 'No data available for correlation calculation'
            }), 400

        correlations = calculate_correlation_matrix(portfolio_data)

        return jsonify({
            'success': True,
            'correlations': correlations
        })

    except Exception as e:
        logger.error(f"Error in get_correlation: {str(e)}", exc_info=True)
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

if __name__ == '__main__':
    app.run(debug=True)

Performance Highlights
Return
Annualized Return

54.17%
+38.20% vs benchmark

Information Ratio

N/A
Alpha

32.62%
Risk
Volatility

43.25%
Value at Risk (5%)

-13.34%
Beta

1.40
Portfolio Growth Summary
$10,000 invested would be worth $86,852 today, representing a total return of 768.52%. Over the same period, the benchmark would be worth $20,957, representing a total return of 109.57%.

The portfolio captured 200.00% of the benchmark's upside while capturing 200.00% of its downside.
Portfolio Performance Statistics
Metric	Current Portfolio	SPY
Start Balance	$10,000.00	$10,000.00
End Balance	$86,851.61	$20,956.94
Annualized Return (CAGR)	54.17%	15.97%
Best Year	323.59%	28.73%
Worst Year	-47.51%	-18.18%
Maximum Drawdown	-55.29%	-33.72%
Sharpe Ratio	1.17	0.72
Sortino Ratio	1.74	1.00
Benchmark Correlation	0.68	N/A
Key Risk and Return Metrics
Metric	Current Portfolio	SPY
Arithmetic Mean (annualized)	69.51%	17.50%
Geometric Mean (annualized)	53.00%	15.67%
Standard Deviation (annualized)	43.25%	20.92%
Downside Deviation (monthly)	9.38%	5.49%
Beta	1.40	N/A
Alpha (annualized)	32.62%	N/A
Historical Value-at-Risk (5%)	-13.34%	-8.25%
Upside Capture Ratio	16830.35%	N/A
Downside Capture Ratio	14413.52%	N/A
Safe Withdrawal Rate	178.85%	265.13%
Positive Periods (%)	5389.51%	5508.74%
Gain/Loss Ratio	1.83	0.99
Starting portfolio comparison...
2024-11-10 03:28:21,125 INFO __main__ Thread-5 (process_request_thread) : Received request data: {
  "portfolios": [
    {
      "name": "Current Portfolio",
      "portfolio": [
        {
          "ticker": "AAPL",
          "weight": 0.5
        },
        {
          "ticker": "TSLA",
          "weight": 0.5
        }
      ]
    },
    {
      "name": "SPY",
      "portfolio": [
        {
          "ticker": "SPY",
          "weight": 1
        }
      ]
    }
  ],
  "startDate": "2019-11-10",
  "transactionCost": 0.1
}
2024-11-10 03:28:21,129 INFO __main__ Thread-5 (process_request_thread) : Fetching data for 3 unique tickers...
C:\Users\calvi\Downloads\top\backend.py:657: FutureWarning: 'M' is deprecated and will be removed in a future version, please use 'ME' instead.
  monthly_returns = weighted_returns.resample('M').apply(
C:\Users\calvi\Downloads\top\backend.py:660: FutureWarning: 'Y' is deprecated and will be removed in a future version, please use 'YE' instead.
  yearly_returns = monthly_returns.resample('Y').apply(
2024-11-10 03:28:21,751 INFO __main__ Thread-5 (process_request_thread) : Returning 1258 dates
2024-11-10 03:28:21,751 INFO __main__ Thread-5 (process_request_thread) : First date: 2019-11-11, Last date: 2024-11-08
2024-11-10 03:28:21,752 INFO __main__ Thread-5 (process_request_thread) : Number of portfolios: 2
2024-11-10 03:28:21,752 INFO __main__ Thread-5 (process_request_thread) : Response prepared successfully
2024-11-10 03:28:21,767 INFO werkzeug Thread-5 (process_request_thread) : 127.0.0.1 - - [10/Nov/2024 03:28:21] "POST /api/compare HTTP/1.1" 200 -
2024-11-10 03:28:21,784 INFO werkzeug Thread-6 (process_request_thread) : 127.0.0.1 - - [10/Nov/2024 03:28:21] "OPTIONS /api/attribution HTTP/1.1" 200 -
2024-11-10 03:28:22,087 INFO werkzeug Thread-7 (process_request_thread) : 127.0.0.1 - - [10/Nov/2024 03:28:22] "OPTIONS /api/correlation HTTP/1.1" 200 -
2024-11-10 03:28:22,100 INFO __main__ Thread-8 (process_request_thread) : Starting attribution analysis...
2024-11-10 03:28:22,101 INFO __main__ Thread-8 (process_request_thread) : Received data: {
  "portfolio": [
    {
      "ticker": "AAPL",
      "weight": 0.5
    },
    {
      "ticker": "TSLA",
      "weight": 0.5
    }
  ],
  "startDate": "2019-11-10",
  "benchmark": "SPY"
}
2024-11-10 03:28:22,105 INFO __main__ Thread-8 (process_request_thread) : Fetching benchmark data...
2024-11-10 03:28:22,107 INFO __main__ Thread-8 (process_request_thread) : Finding common dates...
2024-11-10 03:28:22,109 INFO __main__ Thread-8 (process_request_thread) : Portfolio data shape: (1258, 2)
2024-11-10 03:28:22,109 INFO __main__ Thread-8 (process_request_thread) : Benchmark prices shape: (1258, 1)
2024-11-10 03:28:22,409 INFO __main__ Thread-9 (process_request_thread) : Correlation calculation completed:
2024-11-10 03:28:22,410 INFO werkzeug Thread-9 (process_request_thread) : 127.0.0.1 - - [10/Nov/2024 03:28:22] "POST /api/correlation HTTP/1.1" 200 -
2024-11-10 03:28:22,539 INFO __main__ Thread-8 (process_request_thread) : Attribution calculation completed successfully
2024-11-10 03:28:22,540 INFO __main__ Thread-8 (process_request_thread) : Factor metrics: {'market_beta': 1.399218646133641, 'alpha': 0.3261870660134613, 'r_squared': 0.45821642591084893, 'portfolio_return': 0.5416549403480093, 'benchmark_return': 0.1596978770077644}
2024-11-10 03:28:22,549 INFO werkzeug Thread-8 (process_request_thread) : 127.0.0.1 - - [10/Nov/2024 03:28:22] "POST /api/attribution HTTP/1.1" 200 -