from flask import Flask, request, jsonify
from flask_cors import CORS
import yfinance as yf
import pandas as pd
import numpy as np
from datetime import datetime, timedelta
import json
import os
from scipy import stats
from functools import lru_cache
import hashlib
import pickle
import empyrical as ep
import traceback
import io
import csv

app = Flask(__name__)
CORS(app)

CACHE_DIR = 'cache'
os.makedirs(CACHE_DIR, exist_ok=True)

# Cache functions
def get_cache_key(ticker, start_date):
    return hashlib.md5(f"{ticker}_{start_date}".encode()).hexdigest()

def save_to_cache(key, data):
    cache_file = os.path.join(CACHE_DIR, f"{key}.pkl")
    with open(cache_file, 'wb') as f:
        pickle.dump({'data': data, 'timestamp': datetime.now()}, f)

def load_from_cache(key, max_age_hours=24):
    cache_file = os.path.join(CACHE_DIR, f"{key}.pkl")
    if os.path.exists(cache_file):
        with open(cache_file, 'rb') as f:
            cached = pickle.load(f)
            if datetime.now() - cached['timestamp'] < timedelta(hours=max_age_hours):
                return cached['data']
    return None

# Utility functions
@lru_cache(maxsize=100)
def get_ticker_info(ticker):
    try:
        stock = yf.Ticker(ticker)
        return {
            'sector': stock.info.get('sector', 'Unknown'),
            'industry': stock.info.get('industry', 'Unknown'),
            'name': stock.info.get('longName', ticker)
        }
    except:
        return {'sector': 'Unknown', 'industry': 'Unknown', 'name': ticker}

def validate_ticker(ticker):
    try:
        ticker = ticker.strip()
        if not ticker:
            return False
        stock = yf.Ticker(ticker)
        return bool(stock.info)
    except:
        return False

def fetch_stock_data(ticker, start_date):
    cache_key = get_cache_key(ticker, start_date)
    cached_data = load_from_cache(cache_key)
    
    if cached_data is not None:
        return cached_data
    
    try:
        stock_data = yf.download(ticker, start=start_date, progress=False)
        if not stock_data.empty:
            save_to_cache(cache_key, stock_data)
        return stock_data
    except Exception as e:
        raise Exception(f"Failed to fetch data for {ticker}: {str(e)}")

def clean_for_json(obj):
    if isinstance(obj, (np.integer, np.floating)):
        return float(obj) if not (np.isnan(obj) or np.isinf(obj)) else None
    elif isinstance(obj, dict):
        return {k: clean_for_json(v) for k, v in obj.items()}
    elif isinstance(obj, (list, tuple)):
        return [clean_for_json(x) for x in obj]
    elif isinstance(obj, pd.Series):
        return [clean_for_json(x) for x in obj.values]
    elif isinstance(obj, pd.DataFrame):
        return {col: clean_for_json(obj[col]) for col in obj.columns}
    elif isinstance(obj, np.ndarray):
        return [clean_for_json(x) for x in obj.flatten()]
    elif pd.isna(obj):
        return None
    return obj

# Metric calculation functions
def calculate_sortino_ratio(returns, risk_free_rate=0.02):
    negative_returns = returns[returns < 0]
    if len(negative_returns) == 0:
        return np.inf
    
    downside_std = np.sqrt(np.mean(negative_returns**2))
    excess_return = returns.mean() * 252 - risk_free_rate
    return excess_return / (downside_std * np.sqrt(252)) if downside_std != 0 else np.inf

def calculate_beta(portfolio_returns, market_returns):
    try:
        if isinstance(portfolio_returns, pd.Series):
            portfolio_returns = portfolio_returns.values
        if isinstance(market_returns, pd.Series):
            market_returns = market_returns.values
        elif isinstance(market_returns, pd.DataFrame):
            market_returns = market_returns.iloc[:, 0].values

        portfolio_returns = portfolio_returns.flatten()
        market_returns = market_returns.flatten()

        slope, _, _, _, _ = stats.linregress(market_returns, portfolio_returns)
        return float(slope)
    except Exception as e:
        print(f"Error in calculate_beta: {str(e)}")
        return 1.0

def calculate_alpha(portfolio_returns, market_returns, risk_free_rate=0.02):
    try:
        if isinstance(portfolio_returns, pd.Series):
            portfolio_returns = portfolio_returns.values
        if isinstance(market_returns, pd.Series):
            market_returns = market_returns.values
        elif isinstance(market_returns, pd.DataFrame):
            market_returns = market_returns.iloc[:, 0].values

        portfolio_returns = portfolio_returns.flatten()
        market_returns = market_returns.flatten()

        slope, intercept, _, _, _ = stats.linregress(market_returns, portfolio_returns)
        alpha = intercept + (risk_free_rate * (1 - slope))
        return float(alpha * 252)
    except Exception as e:
        print(f"Error in calculate_alpha: {str(e)}")
        return 0.0

def calculate_tracking_error(portfolio_returns, benchmark_returns):
    try:
        common_index = portfolio_returns.index.intersection(benchmark_returns.index)
        portfolio_returns = portfolio_returns.reindex(common_index)
        benchmark_returns = benchmark_returns.reindex(common_index)
        
        tracking_diff = portfolio_returns - benchmark_returns
        
        tracking_error = np.std(tracking_diff) * np.sqrt(252)
        
        return tracking_error
    except Exception as e:
        print(f"Error calculating tracking error: {str(e)}")
        return 0.0

def calculate_portfolio_metrics(portfolio_data, weights, benchmark_returns=None):
    """Calculate portfolio performance metrics with improved accuracy"""
    try:
        # Constants
        TRADING_DAYS = 252
        RISK_FREE_RATE = 0.02  # 2% annual risk-free rate
        INITIAL_INVESTMENT = 10000

        # Calculate daily returns
        returns = portfolio_data.pct_change().fillna(0)
        portfolio_returns = returns.dot(weights)
        
        # Calculate CAGR and end balance
        start_value = portfolio_data.iloc[0].dot(weights)
        end_value = portfolio_data.iloc[-1].dot(weights)
        years = len(portfolio_returns) / TRADING_DAYS
        
        # Improved CAGR calculation
        total_return = end_value / start_value - 1
        cagr = (1 + total_return) ** (1/years) - 1
        end_balance = INITIAL_INVESTMENT * (1 + total_return)
        
        # Calculate monthly returns properly
        monthly_returns = portfolio_returns.resample('ME').apply(
            lambda x: (1 + x).prod() - 1
        ).dropna()
        
        # Improved mean calculations
        arithmetic_mean_monthly = monthly_returns.mean()
        arithmetic_mean_annual = (1 + arithmetic_mean_monthly)**12 - 1
        
        # Proper geometric mean calculation
        geometric_mean_monthly = (1 + monthly_returns).prod() ** (1/len(monthly_returns)) - 1
        geometric_mean_annual = (1 + geometric_mean_monthly)**12 - 1
        
        # Improved volatility calculations
        daily_std = portfolio_returns.std()
        annual_volatility = daily_std * np.sqrt(TRADING_DAYS)
        
        # Better downside deviation calculation
        downside_returns = monthly_returns[monthly_returns < 0]
        downside_deviation = np.sqrt(np.mean(downside_returns**2)) if len(downside_returns) > 0 else 0
        
        # Improved drawdown calculation
        cumulative_returns = (1 + portfolio_returns).cumprod()
        rolling_max = cumulative_returns.expanding().max()
        drawdowns = cumulative_returns / rolling_max - 1
        max_drawdown = drawdowns.min()
        
        # Improved risk-adjusted return calculations
        excess_return = cagr - RISK_FREE_RATE
        daily_excess_returns = portfolio_returns - RISK_FREE_RATE/TRADING_DAYS
        
        # Better Sharpe ratio calculation
        sharpe_ratio = np.sqrt(TRADING_DAYS) * daily_excess_returns.mean() / daily_std if daily_std != 0 else 0
        
        # Improved Sortino ratio calculation
        downside_std = np.sqrt(np.mean(daily_excess_returns[daily_excess_returns < 0]**2)) * np.sqrt(TRADING_DAYS)
        sortino_ratio = excess_return / downside_std if downside_std != 0 else 0
        
        # Better Calmar ratio calculation
        calmar_ratio = cagr / abs(max_drawdown) if max_drawdown != 0 else np.inf
        
        # Improved VaR calculations
        var_historical = np.percentile(monthly_returns, 5)
        cvar = monthly_returns[monthly_returns <= var_historical].mean()
        
        metrics = {
            'start_balance': INITIAL_INVESTMENT,
            'end_balance': float(end_balance),
            'cagr': float(cagr),
            'annual_volatility': float(annual_volatility),
            'max_drawdown': float(max_drawdown),
            'sharpe_ratio': float(sharpe_ratio),
            'sortino_ratio': float(sortino_ratio),
            'arithmetic_mean_monthly': float(arithmetic_mean_monthly),
            'arithmetic_mean_annual': float(arithmetic_mean_annual),
            'geometric_mean_monthly': float(geometric_mean_monthly),
            'geometric_mean_annual': float(geometric_mean_annual),
            'downside_deviation': float(downside_deviation),
            'var_historical': float(var_historical),
            'cvar': float(cvar),
            'calmar_ratio': float(calmar_ratio)
        }
        
        # Calculate benchmark-relative metrics if benchmark provided
        if benchmark_returns is not None:
            # Align data properly
            common_index = portfolio_returns.index.intersection(benchmark_returns.index)
            portfolio_returns_aligned = portfolio_returns.loc[common_index]
            benchmark_returns_aligned = benchmark_returns.loc[common_index]
            
            # Improved correlation calculation
            correlation = portfolio_returns_aligned.corr(benchmark_returns_aligned)
            
            # Better beta calculation using covariance method
            covariance = np.cov(portfolio_returns_aligned, benchmark_returns_aligned)[0,1]
            benchmark_variance = np.var(benchmark_returns_aligned)
            beta = covariance / benchmark_variance if benchmark_variance != 0 else 1.0
            
            # Calculate benchmark CAGR properly
            benchmark_cumulative = (1 + benchmark_returns_aligned).cumprod()
            benchmark_end_value = benchmark_cumulative.iloc[-1]
            benchmark_cagr = (benchmark_end_value) ** (1/years) - 1
            
            # Improved alpha calculation using CAPM
            alpha = cagr - (RISK_FREE_RATE + beta * (benchmark_cagr - RISK_FREE_RATE))
            
            # Better R-squared calculation
            r_squared = correlation ** 2
            
            # Improved tracking error calculation
            tracking_diff = portfolio_returns_aligned - benchmark_returns_aligned
            tracking_error = tracking_diff.std() * np.sqrt(TRADING_DAYS)
            
            # Better active return calculation
            active_return = cagr - benchmark_cagr
            
            # Improved information ratio
            information_ratio = active_return / tracking_error if tracking_error != 0 else 0
            
            metrics.update({
                'benchmark_correlation': float(correlation),
                'beta': float(beta),
                'alpha': float(alpha),
                'r_squared': float(r_squared),
                'tracking_error': float(tracking_error),
                'information_ratio': float(information_ratio),
                'active_return': float(active_return),
                'benchmark_cagr': float(benchmark_cagr),
                'benchmark_end_balance': float(INITIAL_INVESTMENT * (1 + benchmark_end_value - 1))
            })
        
        return metrics
        
    except Exception as e:
        raise ValueError(f"Error calculating portfolio metrics: {str(e)}")

def save_portfolio(portfolio_data):
    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
    filename = f"portfolio_{timestamp}.json"
    
    os.makedirs('portfolios', exist_ok=True)
    
    filepath = os.path.join('portfolios', filename)
    with open(filepath, 'w') as f:
        json.dump(portfolio_data, f)
    
    return filename

def load_portfolio(filename):
    filepath = os.path.join('portfolios', filename)
    try:
        with open(filepath, 'r') as f:
            return json.load(f)
    except:
        return None

def generate_csv_report(results):
    output = io.StringIO()
    writer = csv.writer(output)
    
    writer.writerow(['Portfolio Metrics'])
    writer.writerow(['Metric', 'Portfolio', 'Benchmark'])
    for key in results['portfolio_metrics']:
        writer.writerow([
            key,
            results['portfolio_metrics'][key],
            results['benchmark_metrics'].get(key, 'N/A')
        ])
    
    writer.writerow([])
    writer.writerow(['Historical Performance'])
    writer.writerow(['Date', 'Portfolio Value', 'Benchmark Value'])
    for i, date in enumerate(results['historical_performance']['dates']):
        writer.writerow([
            date,
            results['historical_performance']['portfolio_values'][i],
            results['historical_performance']['benchmark_values'][i]
        ])
    
    return output.getvalue()

def get_sector_info(ticker):
    try:
        stock = yf.Ticker(ticker)
        return stock.info.get('sector', 'Unknown')
    except:
        return 'Unknown'

def calculate_attribution(portfolio_data, weights, benchmark_data):
    try:
        print("Starting attribution calculation...")
        print(f"Portfolio data shape: {portfolio_data.shape}")
        print(f"Weights length: {len(weights)}")
        print(f"Benchmark data shape: {benchmark_data.shape}")
        
        # Align dates
        common_dates = portfolio_data.index.intersection(benchmark_data.index)
        portfolio_data = portfolio_data.reindex(common_dates)
        benchmark_data = benchmark_data.reindex(common_dates)
        
        # Calculate returns
        portfolio_returns = portfolio_data.pct_change().fillna(0)
        benchmark_returns = pd.Series(benchmark_data.values.flatten(), index=benchmark_data.index).pct_change().fillna(0)
        
        # Calculate weighted returns
        weighted_returns = pd.DataFrame()
        for col, weight in zip(portfolio_returns.columns, weights):
            weighted_returns[col] = portfolio_returns[col] * weight
        
        # Sector Attribution
        sectors = {}
        for ticker in portfolio_data.columns:
            sectors[ticker] = get_sector_info(ticker)
        
        sector_attribution = {}
        total_return = 0
        
        for sector in set(sectors.values()):
            sector_tickers = [t for t, s in sectors.items() if s == sector]
            if sector_tickers:
                # Calculate sector weight
                sector_weights = [weights[list(portfolio_data.columns).index(t)] for t in sector_tickers]
                sector_weight = sum(sector_weights)
                
                # Calculate sector returns
                sector_returns = weighted_returns[sector_tickers].sum(axis=1)
                annual_return = (1 + sector_returns).prod() ** (252/len(sector_returns)) - 1
                contribution = annual_return * sector_weight
                total_return += contribution
                
                sector_attribution[sector] = {
                    'weight': float(sector_weight),
                    'return': float(annual_return),
                    'contribution': float(contribution)
                }
        
        # Factor Attribution
        portfolio_cumulative = (1 + weighted_returns.sum(axis=1)).cumprod()
        benchmark_cumulative = (1 + benchmark_returns).cumprod()
        
        # Calculate annualized returns
        years = (common_dates[-1] - common_dates[0]).days / 365.25
        portfolio_return = (portfolio_cumulative.iloc[-1]) ** (1/years) - 1
        benchmark_return = (benchmark_cumulative.iloc[-1]) ** (1/years) - 1
        
        # Calculate beta and alpha using regression
        model = stats.linregress(benchmark_returns, weighted_returns.sum(axis=1))
        beta = model.slope
        alpha = model.intercept * 252  # Annualize alpha
        r_squared = model.rvalue ** 2
        
        factor_attribution = {
            'market_beta': float(beta),
            'alpha': float(alpha),
            'r_squared': float(r_squared),
            'portfolio_return': float(portfolio_return),
            'benchmark_return': float(benchmark_return)
        }
        
        attribution = {
            'sector': sector_attribution,
            'factor': factor_attribution,
            'total_return': float(total_return)
        }
        
        print("Attribution calculation completed:")
        print(f"Sector attribution: {sector_attribution}")
        print(f"Factor attribution: {factor_attribution}")
        print(f"Total return: {total_return}")
        
        return clean_for_json(attribution)
        
    except Exception as e:
        print(f"Error in calculate_attribution: {str(e)}")
        print(f"Portfolio data shape: {portfolio_data.shape}")
        print(f"Weights length: {len(weights)}")
        print(f"Benchmark data shape: {benchmark_data.shape}")
        traceback.print_exc()
        raise

def calculate_rebalancing_impact(portfolio_data, initial_weights, transaction_cost=0.001, rebalancing_frequency='M'):
    """Calculate the impact of different rebalancing strategies with transaction costs"""
    try:
        initial_investment = 10000
        results = {}
        
        # Define rebalancing frequencies
        frequencies = {
            'null': None,  # Buy and hold
            'M': 'ME',     # Monthly
            'Q': 'QE',     # Quarterly
            'Y': 'YE'      # Yearly
        }
        
        # Calculate daily returns
        daily_returns = portfolio_data.pct_change().fillna(0)
        
        for freq_key, freq in frequencies.items():
            print(f"\nCalculating {freq_key} strategy...")
            
            # Initialize portfolio
            portfolio_value = initial_investment
            current_weights = np.array(initial_weights, dtype=float)
            values = [initial_investment]
            total_cost = 0  # Track total transaction costs
            
            # Initialize positions (number of shares)
            initial_prices = portfolio_data.iloc[0]
            positions = current_weights * initial_investment / initial_prices
            
            # Track daily values
            for i in range(1, len(portfolio_data)):
                # Get current prices
                current_prices = portfolio_data.iloc[i]
                
                # Update positions values
                current_values = positions * current_prices
                portfolio_value = current_values.sum()
                
                # Calculate current weights
                current_weights = current_values / portfolio_value
                
                # Check if rebalancing is needed
                if freq is not None:
                    current_date = portfolio_data.index[i]
                    should_rebalance = False
                    
                    if freq == 'ME' and current_date.is_month_end:
                        should_rebalance = True
                    elif freq == 'QE' and current_date.is_quarter_end:
                        should_rebalance = True
                    elif freq == 'YE' and current_date.is_year_end:
                        should_rebalance = True
                    
                    if should_rebalance:
                        # Calculate rebalancing trades
                        target_values = portfolio_value * initial_weights
                        trades = target_values - current_values
                        
                        # Calculate transaction costs
                        total_trades = np.abs(trades).sum()
                        cost = total_trades * transaction_cost
                        total_cost += cost
                        
                        # Apply transaction costs
                        portfolio_value -= cost
                        
                        # Update positions after costs
                        positions = (target_values / current_prices) * (1 - transaction_cost)
                
                values.append(float(portfolio_value))
            
            # Calculate metrics
            returns = pd.Series(values).pct_change().dropna()
            total_return = values[-1] / values[0] - 1
            years = len(returns) / 252
            
            # Calculate annualized return
            annualized_return = (1 + total_return) ** (1/years) - 1 if years > 0 else 0
            
            # Calculate volatility
            daily_std = returns.std()
            annualized_volatility = daily_std * np.sqrt(252)
            
            # Calculate Sharpe ratio
            excess_returns = returns - 0.02/252  # Daily excess returns over 2% risk-free rate
            sharpe = np.sqrt(252) * excess_returns.mean() / daily_std if daily_std != 0 else 0
            
            # Calculate maximum drawdown
            cumulative = (1 + returns).cumprod()
            rolling_max = cumulative.expanding().max()
            drawdowns = (cumulative - rolling_max) / rolling_max
            max_drawdown = drawdowns.min()
            
            print(f"Strategy: {freq_key}")
            print(f"Final Value: ${portfolio_value:,.2f}")
            print(f"Total Transaction Costs: ${total_cost:,.2f}")
            print(f"Annualized Return: {annualized_return*100:.2f}%")
            print(f"Volatility: {annualized_volatility*100:.2f}%")
            print(f"Max Drawdown: {max_drawdown*100:.2f}%")
            
            results[freq_key] = {
                'final_value': float(values[-1]),
                'annualized_return': float(annualized_return),
                'volatility': float(annualized_volatility),
                'sharpe_ratio': float(sharpe),
                'max_drawdown': float(max_drawdown),
                'values': [float(v) for v in values],
                'total_cost': float(total_cost),
                'cost_percentage': float(total_cost / initial_investment * 100)
            }
        
        return results
        
    except Exception as e:
        print(f"Error in calculate_rebalancing_impact: {str(e)}")
        traceback.print_exc()
        raise

@app.route('/api/validate-ticker', methods=['POST'])
def validate_ticker_endpoint():
    data = request.json
    ticker = data.get('ticker', '')
    
    return jsonify({
        'success': True,
        'valid': validate_ticker(ticker)
    })

@app.route('/api/backtest', methods=['POST'])
def backtest():
    try:
        print("\n=== Starting Backtest ===")
        data = request.json
        print("Request data:", data)
        
        portfolio = data['portfolio']
        start_date = data.get('startDate')
        benchmark = data.get('benchmark', 'SPY')
        
        print(f"Processing portfolio: {portfolio}")
        print(f"Start date: {start_date}")
        print(f"Benchmark: {benchmark}")
        
        # Get portfolio data
        portfolio_data = pd.DataFrame()
        start_date = pd.to_datetime(start_date)
        
        # Fetch data and calculate weights
        stock_data_dict = {}
        weights = []
        for stock in portfolio:
            ticker = stock['ticker'].strip().upper()
            weight = float(stock['weight']) / 100
            
            stock_data = yf.download(ticker, start=start_date, progress=False)
            if stock_data.empty:
                return jsonify({
                    'success': False,
                    'error': f'No data available for {ticker}'
                }), 400
                
            stock_data_dict[ticker] = stock_data['Adj Close']
            weights.append(weight)
            
        # Validate that weights sum to 1
        total_weight = sum(weights)
        if not np.isclose(total_weight, 1.0):
            return jsonify({
                'success': False,
                'error': f'Portfolio weights must sum to 100%. Current sum: {total_weight * 100:.2f}%'
            }), 400
        
        # Fetch benchmark data
        benchmark_data = yf.download(benchmark, start=start_date, progress=False)['Adj Close']
        
        # Create aligned portfolio data
        common_dates = None
        for ticker, data in stock_data_dict.items():
            if common_dates is None:
                common_dates = data.index
            else:
                common_dates = common_dates.intersection(data.index)
        
        # Add benchmark dates to intersection
        common_dates = common_dates.intersection(benchmark_data.index)
        
        # Align all data to common dates
        for ticker in stock_data_dict:
            portfolio_data[ticker] = stock_data_dict[ticker].reindex(common_dates)
        benchmark_data = benchmark_data.reindex(common_dates)
        
        # Drop any remaining NaNs
        portfolio_data = portfolio_data.dropna()
        benchmark_data = benchmark_data.dropna()
        
        if portfolio_data.empty or benchmark_data.empty:
            return jsonify({
                'success': False,
                'error': 'Insufficient overlapping data between portfolio and benchmark.'
            }), 400
        
        # Calculate returns
        portfolio_returns = portfolio_data.pct_change().fillna(0)
        benchmark_returns = benchmark_data.pct_change().fillna(0)  # Already a Series
        
        # Calculate metrics
        portfolio_metrics = calculate_portfolio_metrics(
            portfolio_data=portfolio_data,
            weights=weights,
            benchmark_returns=benchmark_returns
        )
        
        # Create benchmark DataFrame with proper dimensions
        benchmark_df = pd.DataFrame()
        benchmark_df[benchmark] = benchmark_data
        
        benchmark_metrics = calculate_portfolio_metrics(
            portfolio_data=benchmark_df,
            weights=[1.0],
            benchmark_returns=None
        )
        
        # Calculate historical performance
        initial_investment = 10000
        portfolio_values = initial_investment * (1 + portfolio_returns.dot(weights)).cumprod()
        benchmark_values = initial_investment * (1 + benchmark_returns).cumprod()
        
        # Ensure both series have the same index
        common_dates = portfolio_values.index.intersection(benchmark_values.index)
        portfolio_values = portfolio_values.reindex(common_dates)
        benchmark_values = benchmark_values.reindex(common_dates)
        
        response_data = {
            'success': True,
            'portfolio_metrics': clean_for_json(portfolio_metrics),
            'benchmark_metrics': clean_for_json(benchmark_metrics),
            'historical_performance': {
                'dates': common_dates.strftime('%Y-%m-%d').tolist(),
                'portfolio_values': clean_for_json(portfolio_values.values),  # Use .values to get numpy array
                'benchmark_values': clean_for_json(benchmark_values.values),  # Use .values to get numpy array
                'initial_investment': initial_investment
            }
        }
        
        print("Response data prepared successfully")
        return jsonify(response_data)
        
    except Exception as e:
        print(f"\nError in backtest: {str(e)}")
        traceback.print_exc()
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

@app.route('/api/portfolio/save', methods=['POST'])
def save_portfolio_endpoint():
    try:
        portfolio_data = request.json
        filename = save_portfolio(portfolio_data)
        return jsonify({
            'success': True,
            'filename': filename
        })
    except Exception as e:
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

@app.route('/api/portfolio/load', methods=['GET'])
def list_portfolios():
    try:
        os.makedirs('portfolios', exist_ok=True)
        portfolios = []
        for filename in os.listdir('portfolios'):
            if filename.endswith('.json'):
                portfolio = load_portfolio(filename)
                if portfolio:
                    portfolios.append({
                        'filename': filename,
                        'portfolio': portfolio
                    })
        return jsonify({
            'success': True,
            'portfolios': portfolios
        })
    except Exception as e:
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

@app.route('/api/export/csv', methods=['POST'])
def export_csv():
    try:
        results = request.json
        csv_data = generate_csv_report(results)
        return jsonify({
            'success': True,
            'csv_data': csv_data
        })
    except Exception as e:
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

@app.route('/api/attribution', methods=['POST'])
def get_attribution():
    try:
        print("Starting attribution analysis...")
        data = request.json
        portfolio = data['portfolio']
        start_date = pd.to_datetime(data.get('startDate'))
        benchmark = data.get('benchmark', 'SPY')
        
        portfolio_data = pd.DataFrame()
        for stock in portfolio:
            print(f"Fetching data for {stock['ticker']}")
            stock_data = fetch_stock_data(stock['ticker'].strip().upper(), start_date)
            if stock_data.empty:
                return jsonify({
                    'success': False,
                    'error': f'No data available for {stock["ticker"]}'
                }), 400
            portfolio_data[stock['ticker']] = stock_data['Adj Close']
            
        print("Fetching benchmark data...")
        benchmark_data = fetch_stock_data(benchmark, start_date)['Adj Close']
        if benchmark_data.empty:
            return jsonify({
                'success': False,
                'error': f'No data available for benchmark {benchmark}'
            }), 400
        
        print("Finding common dates...")
        common_dates = portfolio_data.index.intersection(benchmark_data.index)
        
        if len(common_dates) == 0:
            return jsonify({
                'success': False,
                'error': 'No overlapping dates found for the selected securities'
            }), 400
            
        portfolio_data = portfolio_data.reindex(common_dates).dropna()
        benchmark_data = benchmark_data.reindex(common_dates).dropna()
        
        if portfolio_data.empty or benchmark_data.empty:
            return jsonify({
                'success': False,
                'error': 'Insufficient overlapping data after reindexing.'
            }), 400
        
        weights = [float(stock['weight']) / 100 for stock in portfolio]
        
        # Validate that weights sum to 1
        total_weight = sum(weights)
        if not np.isclose(total_weight, 1.0):
            return jsonify({
                'success': False,
                'error': f'Portfolio weights must sum to 100%. Current sum: {total_weight * 100:.2f}%'
            }), 400
        
        print("Calculating attribution...")
        try:
            attribution = calculate_attribution(
                portfolio_data=portfolio_data,
                weights=np.array(weights, dtype=float),
                benchmark_data=benchmark_data
            )
            
            return jsonify({
                'success': True,
                'attribution': attribution
            })
        except Exception as calc_error:
            print(f"Attribution calculation error: {str(calc_error)}")
            return jsonify({
                'success': False,
                'error': f'Attribution calculation failed: {str(calc_error)}'
            }), 400
        
    except Exception as e:
        print(f"Error in get_attribution: {str(e)}")
        traceback.print_exc()
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

@app.route('/api/rebalance', methods=['POST'])
def analyze_rebalancing():
    try:
        data = request.json
        portfolio = data['portfolio']
        start_date = data.get('startDate')
        
        portfolio_data = pd.DataFrame()
        weights = []
        
        for stock in portfolio:
            stock_data = fetch_stock_data(stock['ticker'], start_date)
            portfolio_data[stock['ticker']] = stock_data['Adj Close']
            weights.append(float(stock['weight']) / 100)  # Normalize weights
        
        # Validate that weights sum to 1
        total_weight = sum(weights)
        if not np.isclose(total_weight, 1.0):
            return jsonify({
                'success': False,
                'error': f'Portfolio weights must sum to 100%. Current sum: {total_weight * 100:.2f}%'
            }), 400
        
        weights = np.array(weights)
        
        rebalancing_results = calculate_rebalancing_impact(portfolio_data, weights)
        
        return jsonify({
            'success': True,
            'rebalancing_results': rebalancing_results
        })
        
    except Exception as e:
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

@app.route('/api/cache/clear', methods=['POST'])
def clear_cache():
    try:
        for file in os.listdir(CACHE_DIR):
            if file.endswith('.pkl'):
                os.remove(os.path.join(CACHE_DIR, file))
        return jsonify({
            'success': True,
            'message': 'Cache cleared successfully'
        })
    except Exception as e:
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

@app.route('/api/cache/status', methods=['GET'])
def cache_status():
    try:
        cache_files = [f for f in os.listdir(CACHE_DIR) if f.endswith('.pkl')]
        total_size = sum(os.path.getsize(os.path.join(CACHE_DIR, f)) for f in cache_files)
        
        return jsonify({
            'success': True,
            'status': {
                'cached_items': len(cache_files),
                'total_size_mb': total_size / (1024 * 1024),
                'cache_dir': CACHE_DIR
            }
        })
    except Exception as e:
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

def compare_portfolios(portfolios, start_date):
    try:
        results = {}
        all_data = pd.DataFrame()
        
        unique_tickers = set()
        for portfolio in portfolios:
            unique_tickers.update([stock['ticker'] for stock in portfolio['portfolio']])
            
        for ticker in unique_tickers:
            stock_data = fetch_stock_data(ticker, start_date)
            all_data[ticker] = stock_data['Adj Close']
        
        for idx, portfolio_data in enumerate(portfolios):
            portfolio_name = portfolio_data.get('name', f'Portfolio {idx + 1}')
            portfolio_stocks = portfolio_data['portfolio']
            
            tickers = [stock['ticker'] for stock in portfolio_stocks]
            weights = [float(stock['weight']) / 100 for stock in portfolio_stocks]
            portfolio_price_data = all_data[tickers]
            
            portfolio_returns = portfolio_price_data.pct_change().dot(weights)
            portfolio_cumulative = (1 + portfolio_returns).cumprod()
            
            metrics = calculate_portfolio_metrics(portfolio_price_data, weights)
            
            results[portfolio_name] = {
                'metrics': metrics,
                'cumulative_returns': portfolio_cumulative.tolist(),
                'weights': dict(zip(tickers, weights))
            }
        
        return {
            'success': True,
            'comparison_results': results,
            'dates': all_data.index.strftime('%Y-%m-%d').tolist()
        }
        
    except Exception as e:
        return {
            'success': False,
            'error': str(e)
        }

@app.route('/api/compare', methods=['POST'])
def compare_portfolios_endpoint():
    try:
        data = request.json
        portfolios = data['portfolios']
        start_date = data.get('startDate')
        
        results = compare_portfolios(portfolios, start_date)
        return jsonify(results)
        
    except Exception as e:
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

def calculate_correlation_matrix(portfolio_data):
    """Calculate correlation matrix with proper handling of returns"""
    try:
        # Calculate returns first
        returns = portfolio_data.pct_change().dropna()
        
        # Calculate correlations using returns, not prices
        pearson_corr = returns.corr(method='pearson')
        spearman_corr = returns.corr(method='spearman')
        
        # Create mask for diagonal elements
        mask = ~np.eye(len(pearson_corr), dtype=bool)
        
        # Calculate statistics excluding self-correlations
        pearson_stats = {
            'min': float(np.min(pearson_corr.values[mask])),
            'max': float(np.max(pearson_corr.values[mask])),
            'mean': float(np.mean(pearson_corr.values[mask]))
        }
        
        spearman_stats = {
            'min': float(np.min(spearman_corr.values[mask])),
            'max': float(np.max(spearman_corr.values[mask])),
            'mean': float(np.mean(spearman_corr.values[mask]))
        }
        
        correlations = {
            'pearson': {
                'matrix': pearson_corr.to_dict(),
                'min': pearson_stats['min'],
                'max': pearson_stats['max'],
                'mean': pearson_stats['mean']
            },
            'spearman': {
                'matrix': spearman_corr.to_dict(),
                'min': spearman_stats['min'],
                'max': spearman_stats['max'],
                'mean': spearman_stats['mean']
            }
        }
        
        print("Correlation calculation completed:")
        print(f"Pearson stats: {pearson_stats}")
        print(f"Spearman stats: {spearman_stats}")
        
        return clean_for_json(correlations)
        
    except Exception as e:
        print(f"Error in calculate_correlation_matrix: {str(e)}")
        traceback.print_exc()
        raise

@app.route('/api/correlation', methods=['POST'])
def get_correlation():
    try:
        data = request.json
        portfolio = data['portfolio']
        start_date = data.get('startDate')
        
        portfolio_data = pd.DataFrame()
        for stock in portfolio:
            stock_data = fetch_stock_data(stock['ticker'], start_date)
            if stock_data.empty:
                return jsonify({
                    'success': False,
                    'error': f'No data available for {stock["ticker"]}'
                }), 400
            portfolio_data[stock['ticker']] = stock_data['Adj Close']
        
        if portfolio_data.empty:
            return jsonify({
                'success': False,
                'error': 'No data available for correlation calculation'
            }), 400
        
        correlations = calculate_correlation_matrix(portfolio_data)
        
        return jsonify({
            'success': True,
            'correlations': correlations
        })
        
    except Exception as e:
        print(f"Error in get_correlation: {str(e)}")
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

if __name__ == '__main__':
    app.run(debug=True)

enhance and refine my code and fix the below bugs as well:
=== Starting Backtest ===
Request data: {'portfolio': [{'ticker': 'AAPL', 'weight': 40}, {'ticker': 'TSLA', 'weight': 60}], 'benchmark': 'SPY', 'startDate': '2021-11-10'}
Processing portfolio: [{'ticker': 'AAPL', 'weight': 40}, {'ticker': 'TSLA', 'weight': 60}]
Start date: 2021-11-10
Benchmark: SPY

Calculating null strategy...
Starting attribution analysis...
Fetching data for AAPL
Fetching data for TSLA
Strategy: null
Final Value: $10,654.99
Fetching benchmark data...
Total Transaction Costs: $0.00
Annualized Return: 2.15%
Volatility: 38.34%
Max Drawdown: -57.59%

Calculating M strategy...
Finding common dates...
Calculating attribution...
Starting attribution calculation...
Portfolio data shape: (754, 2)
Weights length: 2
Benchmark data shape: (754, 1)
Correlation calculation completed:
Pearson stats: {'min': 0.4271816704882954, 'max': 0.4271816704882954, 'mean': 0.4271816704882954}
Spearman stats: {'min': 0.4748909154847398, 'max': 0.4748909154847398, 'mean': 0.4748909154847398}
127.0.0.1 - - [09/Nov/2024 13:48:10] "POST /api/correlation HTTP/1.1" 200 -
Strategy: M
Final Value: $11,033.87
Total Transaction Costs: $12.68
Annualized Return: 3.35%
Volatility: 41.91%
Max Drawdown: -59.12%

Calculating Q strategy...
Strategy: Q
Final Value: $11,043.55
Total Transaction Costs: $5.60
Annualized Return: 3.38%
Volatility: 40.10%
Max Drawdown: -56.50%

Calculating Y strategy...
Strategy: Y
Final Value: $10,618.30
Total Transaction Costs: $0.14
Annualized Return: 2.03%
Volatility: 38.59%
Max Drawdown: -57.86%
127.0.0.1 - - [09/Nov/2024 13:48:11] "POST /api/rebalance HTTP/1.1" 200 -
Attribution calculation completed:
Sector attribution: {'Consumer Cyclical': {'weight': 0.6, 'return': 0.024285409786969803, 'contribution': 0.01457124587218188}, 'Technology': {'weight': 0.4, 'return': 0.046377464519503864, 'contribution': 0.018550985807801547}}
Factor attribution: {'market_beta': 0.9658725078447199, 'alpha': 0.018259978539356746, 'r_squared': 0.39405934799552206, 'portfolio_return': 0.0532094342983056, 'benchmark_return': 0.09440593980784451}
Total return: 0.033122231679983426
127.0.0.1 - - [09/Nov/2024 13:48:11] "POST /api/attribution HTTP/1.1" 200 -

Error in backtest: Error calculating portfolio metrics: all the input array dimensions except for the concatenation axis must match exactly, but along dimension 1, the array at index 0 has size 754 and the array at index 1 has size 1
Traceback (most recent call last):
  File "C:\Users\calvi\Downloads\react\backend.py", line 252, in calculate_portfolio_metrics
    correlation = portfolio_returns_aligned.corr(benchmark_returns_aligned)
  File "C:\Users\calvi\Downloads\react\venv\lib\site-packages\pandas\core\series.py", line 2977, in corr
    return nanops.nancorr(
  File "C:\Users\calvi\Downloads\react\venv\lib\site-packages\pandas\core\nanops.py", line 85, in _f
    return f(*args, **kwargs)
  File "C:\Users\calvi\Downloads\react\venv\lib\site-packages\pandas\core\nanops.py", line 1615, in nancorr
    return f(a, b)
  File "C:\Users\calvi\Downloads\react\venv\lib\site-packages\pandas\core\nanops.py", line 1638, in func
    return np.corrcoef(a, b)[0, 1]
  File "C:\Users\calvi\Downloads\react\venv\lib\site-packages\numpy\lib\_function_base_impl.py", line 2991, in corrcoef
    c = cov(x, y, rowvar, dtype=dtype)
  File "C:\Users\calvi\Downloads\react\venv\lib\site-packages\numpy\lib\_function_base_impl.py", line 2783, in cov
    X = np.concatenate((X, y), axis=0)
ValueError: all the input array dimensions except for the concatenation axis must match exactly, but along dimension 1, the array at index 0 has size 754 and the array at index 1 has size 1

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\calvi\Downloads\react\backend.py", line 648, in backtest
    portfolio_metrics = calculate_portfolio_metrics(
  File "C:\Users\calvi\Downloads\react\backend.py", line 295, in calculate_portfolio_metrics
    raise ValueError(f"Error calculating portfolio metrics: {str(e)}")
ValueError: Error calculating portfolio metrics: all the input array dimensions except for the concatenation axis must match exactly, but along dimension 1, the array at index 0 has size 754 and the array at index 1 has size 1
127.0.0.1 - - [09/Nov/2024 13:48:11] "POST /api/backtest HTTP/1.1" 400 -