Rebalancing Strategy Comparison
Strategy	Final Value	Annual Return	Volatility
Monthly Rebalancing	$3,571,384.22	224.93%	4428.66%
Quarterly Rebalancing	$3,571,384.22	224.93%	4428.66%
Yearly Rebalancing	$3,571,384.22	224.93%	4428.66%
Buy and Hold	$3,571,384.22	224.93%	4428.66%
Strategy Performance Comparison
2020-04-06
2021-01-04
2021-10-01
2022-07-01
2023-03-31
2023-12-29
2024-11-08
Date
$0
$950,000
$1,900,000
$2,850,000
$3,800,000
Portfolio Value ($)
Buy and HoldMonthlyQuarterlyYearly

Comparison of different rebalancing strategies starting with the same initial investment.
=== Starting Backtest ===
Request data: {'portfolio': [{'ticker': 'AAPL', 'weight': 100}], 'benchmark': 'SPY', 'startDate': '2019-11-11'}
Processing portfolio: [{'ticker': 'AAPL', 'weight': 100}]
Start date: 2019-11-11
Benchmark: SPY
127.0.0.1 - - [09/Nov/2024 12:38:26] "POST /api/correlation HTTP/1.1" 200 -
Starting attribution analysis...
Fetching data for AAPL

Calculating null strategy...
Fetching benchmark data...
Finding common dates...
Calculating attribution...
Starting attribution calculation...
Portfolio data shape: (1258, 1)
Weights length: 1
Benchmark data shape: (1258, 1)
Final value: 3571384.219981531
Annualized return: 2.249270389207252
Volatility: 44.28655137901681
Sharpe Ratio: 0.4541914436963586
Sortino Ratio: 7.13225255515164
Max Drawdown: -0.31427264769904706

Calculating M strategy...
Final value: 3571384.2199815284
Annualized return: 2.249270389207251
Volatility: 44.28655137901681
Sharpe Ratio: 0.4541914436963586
Sortino Ratio: 7.132252555151637
Max Drawdown: -0.31427264769904706

Calculating Q strategy...
Final value: 3571384.219981531
Annualized return: 2.249270389207252
Volatility: 44.28655137901681
Sharpe Ratio: 0.4541914436963586
Sortino Ratio: 7.13225255515164
Max Drawdown: -0.31427264769904706

Calculating Y strategy...
Final value: 3571384.219981531
Annualized return: 2.249270389207252
Volatility: 44.28655137901681
Sharpe Ratio: 0.4541914436963586
Sortino Ratio: 7.13225255515164
Max Drawdown: -0.31427264769904706
127.0.0.1 - - [09/Nov/2024 12:38:27] "POST /api/rebalance HTTP/1.1" 200 -
Response data prepared successfully
127.0.0.1 - - [09/Nov/2024 12:38:28] "POST /api/backtest HTTP/1.1" 200 -
Attribution calculation completed:
Sector attribution: {'Technology': {'weight': 1.0, 'return': 0.290455692607696, 'contribution': 0.290455692607696}}
Factor attribution: {'market_beta': 1.0, 'alpha': 0.0, 'r_squared': 1.0, 'portfolio_return': 0.290455692607696, 'benchmark_return': 0.290455692607696}
Total return: 0.290455692607696
127.0.0.1 - - [09/Nov/2024 12:38:28] "POST /api/attribution HTTP/1.1" 200 -


fix the above bugs

from flask import Flask, request, jsonify
from flask_cors import CORS
import yfinance as yf
import pandas as pd
import numpy as np
from datetime import datetime, timedelta
import json
import os
from scipy import stats
from functools import lru_cache
import hashlib
import pickle
import empyrical as ep
import traceback
import io
import csv

app = Flask(__name__)
CORS(app)

CACHE_DIR = 'cache'
os.makedirs(CACHE_DIR, exist_ok=True)

# Cache functions
def get_cache_key(ticker, start_date):
    return hashlib.md5(f"{ticker}_{start_date}".encode()).hexdigest()

def save_to_cache(key, data):
    cache_file = os.path.join(CACHE_DIR, f"{key}.pkl")
    with open(cache_file, 'wb') as f:
        pickle.dump({'data': data, 'timestamp': datetime.now()}, f)

def load_from_cache(key, max_age_hours=24):
    cache_file = os.path.join(CACHE_DIR, f"{key}.pkl")
    if os.path.exists(cache_file):
        with open(cache_file, 'rb') as f:
            cached = pickle.load(f)
            if datetime.now() - cached['timestamp'] < timedelta(hours=max_age_hours):
                return cached['data']
    return None

# Utility functions
@lru_cache(maxsize=100)
def get_ticker_info(ticker):
    try:
        stock = yf.Ticker(ticker)
        return {
            'sector': stock.info.get('sector', 'Unknown'),
            'industry': stock.info.get('industry', 'Unknown'),
            'name': stock.info.get('longName', ticker)
        }
    except:
        return {'sector': 'Unknown', 'industry': 'Unknown', 'name': ticker}

def validate_ticker(ticker):
    try:
        ticker = ticker.strip()
        if not ticker:
            return False
        stock = yf.Ticker(ticker)
        return bool(stock.info)
    except:
        return False

def fetch_stock_data(ticker, start_date):
    cache_key = get_cache_key(ticker, start_date)
    cached_data = load_from_cache(cache_key)
    
    if cached_data is not None:
        return cached_data
    
    try:
        stock_data = yf.download(ticker, start=start_date, progress=False)
        if not stock_data.empty:
            save_to_cache(cache_key, stock_data)
        return stock_data
    except Exception as e:
        raise Exception(f"Failed to fetch data for {ticker}: {str(e)}")

def clean_for_json(obj):
    if isinstance(obj, (np.integer, np.floating)):
        return float(obj) if not (np.isnan(obj) or np.isinf(obj)) else None
    elif isinstance(obj, dict):
        return {k: clean_for_json(v) for k, v in obj.items()}
    elif isinstance(obj, (list, tuple)):
        return [clean_for_json(x) for x in obj]
    elif isinstance(obj, pd.Series):
        return [clean_for_json(x) for x in obj.values]
    elif isinstance(obj, pd.DataFrame):
        return {col: clean_for_json(obj[col]) for col in obj.columns}
    elif isinstance(obj, np.ndarray):
        return [clean_for_json(x) for x in obj.flatten()]
    elif pd.isna(obj):
        return None
    return obj

# Metric calculation functions
def calculate_sortino_ratio(returns, risk_free_rate=0.02):
    negative_returns = returns[returns < 0]
    if len(negative_returns) == 0:
        return np.inf
    
    downside_std = np.sqrt(np.mean(negative_returns**2))
    excess_return = returns.mean() * 252 - risk_free_rate
    return excess_return / (downside_std * np.sqrt(252)) if downside_std != 0 else np.inf

def calculate_beta(portfolio_returns, market_returns):
    try:
        if isinstance(portfolio_returns, pd.Series):
            portfolio_returns = portfolio_returns.values
        if isinstance(market_returns, pd.Series):
            market_returns = market_returns.values
        elif isinstance(market_returns, pd.DataFrame):
            market_returns = market_returns.iloc[:, 0].values

        portfolio_returns = portfolio_returns.flatten()
        market_returns = market_returns.flatten()

        slope, _, _, _, _ = stats.linregress(market_returns, portfolio_returns)
        return float(slope)
    except Exception as e:
        print(f"Error in calculate_beta: {str(e)}")
        return 1.0

def calculate_alpha(portfolio_returns, market_returns, risk_free_rate=0.02):
    try:
        if isinstance(portfolio_returns, pd.Series):
            portfolio_returns = portfolio_returns.values
        if isinstance(market_returns, pd.Series):
            market_returns = market_returns.values
        elif isinstance(market_returns, pd.DataFrame):
            market_returns = market_returns.iloc[:, 0].values

        portfolio_returns = portfolio_returns.flatten()
        market_returns = market_returns.flatten()

        slope, intercept, _, _, _ = stats.linregress(market_returns, portfolio_returns)
        alpha = intercept + (risk_free_rate * (1 - slope))
        return float(alpha * 252)
    except Exception as e:
        print(f"Error in calculate_alpha: {str(e)}")
        return 0.0

def calculate_tracking_error(portfolio_returns, benchmark_returns):
    try:
        common_index = portfolio_returns.index.intersection(benchmark_returns.index)
        portfolio_returns = portfolio_returns.reindex(common_index)
        benchmark_returns = benchmark_returns.reindex(common_index)
        
        tracking_diff = portfolio_returns - benchmark_returns
        
        tracking_error = np.std(tracking_diff) * np.sqrt(252)
        
        return tracking_error
    except Exception as e:
        print(f"Error calculating tracking error: {str(e)}")
        return 0.0

def calculate_portfolio_metrics(portfolio_data, weights, benchmark_returns=None):
    """Calculate portfolio performance metrics"""
    try:
        # Constants
        TRADING_DAYS = 252
        RISK_FREE_RATE = 0.02
        INITIAL_INVESTMENT = 10000

        # Calculate returns and values
        returns = portfolio_data.pct_change().fillna(0)
        portfolio_returns = returns.dot(weights)
        
        # Calculate CAGR
        start_value = portfolio_data.iloc[0].dot(weights)
        end_value = portfolio_data.iloc[-1].dot(weights)
        years = len(portfolio_returns) / TRADING_DAYS
        cagr = (end_value / start_value) ** (1/years) - 1
        
        # Monthly metrics
        monthly_returns = portfolio_returns.resample('ME').apply(
            lambda x: (1 + x).prod() - 1
        ).dropna()
        
        # Calculate means
        arithmetic_mean_monthly = monthly_returns.mean()
        arithmetic_mean_annual = (1 + arithmetic_mean_monthly)**12 - 1
        geometric_mean_monthly = (1 + monthly_returns).prod() ** (1/len(monthly_returns)) - 1
        geometric_mean_annual = (1 + geometric_mean_monthly)**12 - 1
        
        # Risk metrics
        annual_volatility = portfolio_returns.std() * np.sqrt(TRADING_DAYS)
        downside_returns = monthly_returns[monthly_returns < 0]
        downside_deviation = np.sqrt(np.mean(downside_returns**2)) if len(downside_returns) > 0 else 0
        
        # Drawdown analysis
        cumulative_returns = (1 + portfolio_returns).cumprod()
        rolling_max = cumulative_returns.expanding().max()
        drawdowns = (cumulative_returns - rolling_max) / rolling_max
        max_drawdown = drawdowns.min()
        
        # Risk-adjusted returns
        excess_return = cagr - RISK_FREE_RATE
        sharpe_ratio = excess_return / annual_volatility if annual_volatility != 0 else 0
        sortino_ratio = excess_return / (downside_deviation * np.sqrt(12)) if downside_deviation != 0 else 0
        calmar_ratio = cagr / abs(max_drawdown) if max_drawdown != 0 else 0
        
        # Value at Risk
        var_historical = np.percentile(monthly_returns, 5)
        cvar = monthly_returns[monthly_returns <= var_historical].mean()
        
        # Base metrics dictionary
        metrics = {
            'start_balance': INITIAL_INVESTMENT,
            'end_balance': float(INITIAL_INVESTMENT * (1 + (end_value/start_value - 1))),
            'cagr': float(cagr),
            'annual_volatility': float(annual_volatility),
            'max_drawdown': float(max_drawdown),
            'sharpe_ratio': float(sharpe_ratio),
            'sortino_ratio': float(sortino_ratio),
            'arithmetic_mean_monthly': float(arithmetic_mean_monthly),
            'arithmetic_mean_annual': float(arithmetic_mean_annual),
            'geometric_mean_monthly': float(geometric_mean_monthly),
            'geometric_mean_annual': float(geometric_mean_annual),
            'downside_deviation': float(downside_deviation),
            'var_historical': float(var_historical),
            'cvar': float(cvar),
            'calmar_ratio': float(calmar_ratio)
        }
        
        # Add benchmark-relative metrics if benchmark provided
        if benchmark_returns is not None:
            # Align data
            common_index = portfolio_returns.index.intersection(benchmark_returns.index)
            portfolio_returns_aligned = portfolio_returns.reindex(common_index)
            benchmark_returns_aligned = benchmark_returns.reindex(common_index)
            
            # Calculate correlation and beta
            correlation = portfolio_returns_aligned.corr(benchmark_returns_aligned)
            beta = correlation * (portfolio_returns_aligned.std() / benchmark_returns_aligned.std())
            
            # Calculate benchmark CAGR
            benchmark_cumulative = (1 + benchmark_returns_aligned).cumprod()
            benchmark_cagr = (benchmark_cumulative.iloc[-1]) ** (1/years) - 1
            
            # Calculate alpha using CAPM
            alpha = cagr - (RISK_FREE_RATE + beta * (benchmark_cagr - RISK_FREE_RATE))
            
            # Calculate R-squared
            r_squared = correlation ** 2
            
            # Calculate tracking error and information ratio
            tracking_error = np.std(portfolio_returns_aligned - benchmark_returns_aligned) * np.sqrt(TRADING_DAYS)
            active_return = cagr - benchmark_cagr
            information_ratio = active_return / tracking_error if tracking_error != 0 else 0
            
            # Calculate capture ratios
            up_months = benchmark_returns_aligned > 0
            down_months = benchmark_returns_aligned < 0
            
            up_capture = (np.mean(portfolio_returns_aligned[up_months]) / 
                        np.mean(benchmark_returns_aligned[up_months])) if any(up_months) else 0
            down_capture = (np.mean(portfolio_returns_aligned[down_months]) / 
                          np.mean(benchmark_returns_aligned[down_months])) if any(down_months) else 0
            
            metrics.update({
                'benchmark_correlation': float(correlation),
                'beta': float(beta),
                'alpha': float(alpha),
                'r_squared': float(r_squared),
                'tracking_error': float(tracking_error),
                'information_ratio': float(information_ratio),
                'active_return': float(active_return),
                'up_capture': float(up_capture * 100),
                'down_capture': float(down_capture * 100)
            })
        
        # Ensure no NaN or infinite values
        for key in metrics:
            if isinstance(metrics[key], float):
                if np.isnan(metrics[key]) or np.isinf(metrics[key]):
                    metrics[key] = None
        
        return metrics
        
    except Exception as e:
        raise ValueError(f"Error calculating portfolio metrics: {str(e)}")

def save_portfolio(portfolio_data):
    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
    filename = f"portfolio_{timestamp}.json"
    
    os.makedirs('portfolios', exist_ok=True)
    
    filepath = os.path.join('portfolios', filename)
    with open(filepath, 'w') as f:
        json.dump(portfolio_data, f)
    
    return filename

def load_portfolio(filename):
    filepath = os.path.join('portfolios', filename)
    try:
        with open(filepath, 'r') as f:
            return json.load(f)
    except:
        return None

def generate_csv_report(results):
    output = io.StringIO()
    writer = csv.writer(output)
    
    writer.writerow(['Portfolio Metrics'])
    writer.writerow(['Metric', 'Portfolio', 'Benchmark'])
    for key in results['portfolio_metrics']:
        writer.writerow([
            key,
            results['portfolio_metrics'][key],
            results['benchmark_metrics'].get(key, 'N/A')
        ])
    
    writer.writerow([])
    writer.writerow(['Historical Performance'])
    writer.writerow(['Date', 'Portfolio Value', 'Benchmark Value'])
    for i, date in enumerate(results['historical_performance']['dates']):
        writer.writerow([
            date,
            results['historical_performance']['portfolio_values'][i],
            results['historical_performance']['benchmark_values'][i]
        ])
    
    return output.getvalue()

def get_sector_info(ticker):
    try:
        stock = yf.Ticker(ticker)
        return stock.info.get('sector', 'Unknown')
    except:
        return 'Unknown'

def calculate_attribution(portfolio_data, weights, benchmark_data):
    try:
        print("Starting attribution calculation...")
        print(f"Portfolio data shape: {portfolio_data.shape}")
        print(f"Weights length: {len(weights)}")
        print(f"Benchmark data shape: {benchmark_data.shape}")
        
        # Align dates
        common_dates = portfolio_data.index.intersection(benchmark_data.index)
        portfolio_data = portfolio_data.reindex(common_dates)
        benchmark_data = benchmark_data.reindex(common_dates)
        
        # Calculate returns
        portfolio_returns = portfolio_data.pct_change().fillna(0)
        benchmark_returns = pd.Series(benchmark_data.values.flatten(), index=benchmark_data.index).pct_change().fillna(0)
        
        # Calculate weighted returns
        weighted_returns = pd.DataFrame()
        for col, weight in zip(portfolio_returns.columns, weights):
            weighted_returns[col] = portfolio_returns[col] * weight
        
        # Sector Attribution
        sectors = {}
        for ticker in portfolio_data.columns:
            sectors[ticker] = get_sector_info(ticker)
        
        sector_attribution = {}
        total_return = 0
        
        for sector in set(sectors.values()):
            sector_tickers = [t for t, s in sectors.items() if s == sector]
            if sector_tickers:
                # Calculate sector weight
                sector_weights = [weights[list(portfolio_data.columns).index(t)] for t in sector_tickers]
                sector_weight = sum(sector_weights)
                
                # Calculate sector returns
                sector_returns = weighted_returns[sector_tickers].sum(axis=1)
                annual_return = (1 + sector_returns).prod() ** (252/len(sector_returns)) - 1
                contribution = annual_return * sector_weight
                total_return += contribution
                
                sector_attribution[sector] = {
                    'weight': float(sector_weight),
                    'return': float(annual_return),
                    'contribution': float(contribution)
                }
        
        # Factor Attribution
        portfolio_cumulative = (1 + weighted_returns.sum(axis=1)).cumprod()
        benchmark_cumulative = (1 + benchmark_returns).cumprod()
        
        # Calculate annualized returns
        years = len(portfolio_returns) / 252
        portfolio_return = (portfolio_cumulative.iloc[-1]) ** (1/years) - 1
        benchmark_return = (benchmark_cumulative.iloc[-1]) ** (1/years) - 1
        
        # Calculate beta and alpha using regression
        model = stats.linregress(benchmark_returns, weighted_returns.sum(axis=1))
        beta = model.slope
        alpha = model.intercept * 252  # Annualize alpha
        r_squared = model.rvalue ** 2
        
        factor_attribution = {
            'market_beta': float(beta),
            'alpha': float(alpha),
            'r_squared': float(r_squared),
            'portfolio_return': float(portfolio_return),
            'benchmark_return': float(benchmark_return)
        }
        
        attribution = {
            'sector': sector_attribution,
            'factor': factor_attribution,
            'total_return': float(total_return)
        }
        
        print("Attribution calculation completed:")
        print(f"Sector attribution: {sector_attribution}")
        print(f"Factor attribution: {factor_attribution}")
        print(f"Total return: {total_return}")
        
        return clean_for_json(attribution)
        
    except Exception as e:
        print(f"Error in calculate_attribution: {str(e)}")
        print(f"Portfolio data shape: {portfolio_data.shape}")
        print(f"Weights length: {len(weights)}")
        print(f"Benchmark data shape: {benchmark_data.shape}")
        import traceback
        traceback.print_exc()
        raise

def calculate_rebalancing_impact(portfolio_data, initial_weights, rebalancing_frequency='M'):
    """Calculate the impact of different rebalancing strategies"""
    try:
        initial_investment = 10000
        results = {}
        
        # Define rebalancing frequencies
        frequencies = {
            'null': None,  # Buy and hold
            'M': 'ME',     # Monthly (end of month)
            'Q': 'QE',     # Quarterly (end of quarter)
            'Y': 'YE'      # Yearly (end of year)
        }
        
        # Calculate daily returns
        daily_returns = portfolio_data.pct_change().fillna(0)
        
        for freq_key, freq in frequencies.items():
            print(f"\nCalculating {freq_key} strategy...")
            
            # Initialize portfolio
            portfolio_value = initial_investment
            current_weights = np.array(initial_weights, dtype=float)
            values = [initial_investment]
            
            # Initialize positions (number of shares)
            initial_prices = portfolio_data.iloc[0]
            positions = (current_weights * initial_investment) / initial_prices
            
            # Track daily values
            for i in range(1, len(portfolio_data)):
                # Get current prices
                current_prices = portfolio_data.iloc[i]
                
                # Calculate current portfolio value
                portfolio_value = (positions * current_prices).sum()
                values.append(float(portfolio_value))
                
                # Calculate current weights
                if portfolio_value != 0:
                    current_weights = (positions * current_prices) / portfolio_value
                else:
                    current_weights = current_weights  # Retain previous weights if portfolio_value is 0
                
                # Check if rebalancing is needed
                if freq is not None:
                    current_date = portfolio_data.index[i]
                    should_rebalance = False
                    
                    if freq == 'ME' and current_date.is_month_end:
                        should_rebalance = True
                    elif freq == 'QE' and current_date.is_quarter_end:
                        should_rebalance = True
                    elif freq == 'YE' and current_date.is_year_end:
                        should_rebalance = True
                    
                    if should_rebalance:
                        # Rebalance to target weights without altering the portfolio value
                        if portfolio_value > 0:
                            positions = (current_weights * portfolio_value) / current_prices
                        else:
                            positions = np.zeros_like(positions)
            
            # Calculate metrics
            returns = pd.Series(values).pct_change().dropna()
            total_return = values[-1] / values[0] - 1
            years = len(returns) / 252  # Approximate number of trading days in a year
            
            # Handle edge cases for annualized_return
            if (1 + total_return) <= 0:
                annualized_return = -1.0  # Maximum loss scenario
            else:
                try:
                    annualized_return = (1 + total_return) ** (1/years) - 1
                except:
                    annualized_return = 0.0  # Fallback to 0 if calculation fails
                
            # Calculate volatility
            volatility = returns.std() * np.sqrt(252)
            
            # Calculate Sharpe ratio
            excess_returns = returns - (0.02 / 252)  # Daily excess returns over 2% risk-free rate
            sharpe = (np.sqrt(252) * excess_returns.mean()) / returns.std() if returns.std() != 0 else 0
            
            # Calculate Sortino ratio
            downside_returns = returns[returns < 0]
            downside_std = np.sqrt(np.mean(downside_returns**2)) * np.sqrt(252) if len(downside_returns) > 0 else 0
            sortino = (annualized_return - 0.02) / downside_std if downside_std != 0 else 0
            
            # Calculate maximum drawdown
            cumulative = (1 + returns).cumprod()
            rolling_max = cumulative.expanding().max()
            drawdowns = (cumulative - rolling_max) / rolling_max
            max_drawdown = drawdowns.min()
            
            # Debugging Statements
            print(f"Final value: {values[-1]}")
            print(f"Annualized return: {annualized_return}")
            print(f"Volatility: {volatility}")
            print(f"Sharpe Ratio: {sharpe}")
            print(f"Sortino Ratio: {sortino}")
            print(f"Max Drawdown: {max_drawdown}")
            
            results[freq_key] = {
                'final_value': float(values[-1]),
                'annualized_return': float(annualized_return),
                'volatility': float(volatility),
                'sharpe_ratio': float(sharpe),
                'sortino_ratio': float(sortino),
                'max_drawdown': float(max_drawdown),
                'values': [float(v) for v in values]
            }
        
        return results
    
    except Exception as e:
        print(f"Error in calculate_rebalancing_impact: {str(e)}")
        traceback.print_exc()
        raise

@app.route('/api/validate-ticker', methods=['POST'])
def validate_ticker_endpoint():
    data = request.json
    ticker = data.get('ticker', '')
    
    return jsonify({
        'success': True,
        'valid': validate_ticker(ticker)
    })

@app.route('/api/backtest', methods=['POST'])
def backtest():
    try:
        print("\n=== Starting Backtest ===")
        data = request.json
        print("Request data:", data)
        
        portfolio = data['portfolio']
        start_date = data.get('startDate')
        benchmark = data.get('benchmark', 'SPY')
        
        print(f"Processing portfolio: {portfolio}")
        print(f"Start date: {start_date}")
        print(f"Benchmark: {benchmark}")
        
        # Get portfolio data
        portfolio_data = pd.DataFrame()
        start_date = pd.to_datetime(start_date)
        
        # Fetch data and calculate weights
        stock_data_dict = {}
        weights = []
        for stock in portfolio:
            ticker = stock['ticker'].strip().upper()
            weight = float(stock['weight']) / 100
            
            stock_data = yf.download(ticker, start=start_date, progress=False)
            if stock_data.empty:
                return jsonify({
                    'success': False,
                    'error': f'No data available for {ticker}'
                }), 400
                
            stock_data_dict[ticker] = stock_data['Adj Close']
            weights.append(weight)
            
        # Fetch benchmark data
        benchmark_data = yf.download(benchmark, start=start_date, progress=False)['Adj Close']
        
        # Align dates
        all_series = list(stock_data_dict.values()) + [benchmark_data]
        common_dates = pd.Index.intersection(*[s.index for s in all_series])
        
        # Create aligned portfolio data
        for ticker in stock_data_dict:
            portfolio_data[ticker] = stock_data_dict[ticker].reindex(common_dates)
        benchmark_data = benchmark_data.reindex(common_dates)
        
        # Calculate returns
        portfolio_returns = portfolio_data.pct_change()
        benchmark_returns = pd.Series(benchmark_data.values.flatten(), index=benchmark_data.index).pct_change()
        
        # Calculate metrics
        portfolio_metrics = calculate_portfolio_metrics(
            portfolio_data=portfolio_data,
            weights=weights,
            benchmark_returns=benchmark_returns
        )
        
        benchmark_df = pd.DataFrame({benchmark: benchmark_data.values.flatten()}, index=common_dates)
        benchmark_metrics = calculate_portfolio_metrics(
            portfolio_data=benchmark_df,
            weights=[1.0],
            benchmark_returns=None
        )
        
        # Calculate historical performance
        initial_investment = 10000
        
        # Calculate portfolio values
        portfolio_values = initial_investment * (1 + portfolio_returns).dot(weights).cumprod()
        
        # Calculate benchmark values
        benchmark_values = initial_investment * (1 + benchmark_returns).cumprod()
        
        response_data = {
            'success': True,
            'portfolio_metrics': clean_for_json(portfolio_metrics),
            'benchmark_metrics': clean_for_json(benchmark_metrics),
            'historical_performance': {
                'dates': common_dates.strftime('%Y-%m-%d').tolist(),
                'portfolio_values': clean_for_json(portfolio_values),
                'benchmark_values': clean_for_json(benchmark_values),
                'initial_investment': initial_investment
            }
        }
        
        print("Response data prepared successfully")
        return jsonify(response_data)
        
    except Exception as e:
        print(f"\nError in backtest: {str(e)}")
        import traceback
        traceback.print_exc()
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

@app.route('/api/portfolio/save', methods=['POST'])
def save_portfolio_endpoint():
    try:
        portfolio_data = request.json
        filename = save_portfolio(portfolio_data)
        return jsonify({
            'success': True,
            'filename': filename
        })
    except Exception as e:
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

@app.route('/api/portfolio/load', methods=['GET'])
def list_portfolios():
    try:
        os.makedirs('portfolios', exist_ok=True)
        portfolios = []
        for filename in os.listdir('portfolios'):
            if filename.endswith('.json'):
                portfolio = load_portfolio(filename)
                if portfolio:
                    portfolios.append({
                        'filename': filename,
                        'portfolio': portfolio
                    })
        return jsonify({
            'success': True,
            'portfolios': portfolios
        })
    except Exception as e:
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

@app.route('/api/export/csv', methods=['POST'])
def export_csv():
    try:
        results = request.json
        csv_data = generate_csv_report(results)
        return jsonify({
            'success': True,
            'csv_data': csv_data
        })
    except Exception as e:
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

@app.route('/api/attribution', methods=['POST'])
def get_attribution():
    try:
        print("Starting attribution analysis...")
        data = request.json
        portfolio = data['portfolio']
        start_date = pd.to_datetime(data.get('startDate'))
        benchmark = data.get('benchmark', 'SPY')
        
        portfolio_data = pd.DataFrame()
        for stock in portfolio:
            print(f"Fetching data for {stock['ticker']}")
            stock_data = fetch_stock_data(stock['ticker'].strip().upper(), start_date)
            if stock_data.empty:
                return jsonify({
                    'success': False,
                    'error': f'No data available for {stock["ticker"]}'
                }), 400
            portfolio_data[stock['ticker']] = stock_data['Adj Close']
            
        print("Fetching benchmark data...")
        benchmark_data = fetch_stock_data(benchmark, start_date)['Adj Close']
        if benchmark_data.empty:
            return jsonify({
                'success': False,
                'error': f'No data available for benchmark {benchmark}'
            }), 400
        
        print("Finding common dates...")
        common_dates = portfolio_data.index.intersection(benchmark_data.index)
        
        if len(common_dates) == 0:
            return jsonify({
                'success': False,
                'error': 'No overlapping dates found for the selected securities'
            }), 400
            
        portfolio_data = portfolio_data.reindex(common_dates)
        benchmark_data = benchmark_data.reindex(common_dates)
        
        weights = [float(stock['weight']) / 100 for stock in portfolio]
        
        print("Calculating attribution...")
        try:
            attribution = calculate_attribution(
                portfolio_data=portfolio_data,
                weights=np.array(weights, dtype=float),
                benchmark_data=benchmark_data
            )
            
            return jsonify({
                'success': True,
                'attribution': attribution
            })
        except Exception as calc_error:
            print(f"Attribution calculation error: {str(calc_error)}")
            return jsonify({
                'success': False,
                'error': f'Attribution calculation failed: {str(calc_error)}'
            }), 400
        
    except Exception as e:
        print(f"Error in get_attribution: {str(e)}")
        import traceback
        traceback.print_exc()
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

@app.route('/api/rebalance', methods=['POST'])
def analyze_rebalancing():
    try:
        data = request.json
        portfolio = data['portfolio']
        start_date = data.get('startDate')
        
        portfolio_data = pd.DataFrame()
        weights = []
        
        for stock in portfolio:
            stock_data = fetch_stock_data(stock['ticker'], start_date)
            portfolio_data[stock['ticker']] = stock_data['Adj Close']
            weights.append(stock['weight'])
        
        weights = np.array(weights)
        
        rebalancing_results = calculate_rebalancing_impact(portfolio_data, weights)
        
        return jsonify({
            'success': True,
            'rebalancing_results': rebalancing_results
        })
        
    except Exception as e:
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

@app.route('/api/cache/clear', methods=['POST'])
def clear_cache():
    try:
        for file in os.listdir(CACHE_DIR):
            if file.endswith('.pkl'):
                os.remove(os.path.join(CACHE_DIR, file))
        return jsonify({
            'success': True,
            'message': 'Cache cleared successfully'
        })
    except Exception as e:
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

@app.route('/api/cache/status', methods=['GET'])
def cache_status():
    try:
        cache_files = [f for f in os.listdir(CACHE_DIR) if f.endswith('.pkl')]
        total_size = sum(os.path.getsize(os.path.join(CACHE_DIR, f)) for f in cache_files)
        
        return jsonify({
            'success': True,
            'status': {
                'cached_items': len(cache_files),
                'total_size_mb': total_size / (1024 * 1024),
                'cache_dir': CACHE_DIR
            }
        })
    except Exception as e:
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

def compare_portfolios(portfolios, start_date):
    try:
        results = {}
        all_data = pd.DataFrame()
        
        unique_tickers = set()
        for portfolio in portfolios:
            unique_tickers.update([stock['ticker'] for stock in portfolio['portfolio']])
            
        for ticker in unique_tickers:
            stock_data = fetch_stock_data(ticker, start_date)
            all_data[ticker] = stock_data['Adj Close']
        
        for idx, portfolio_data in enumerate(portfolios):
            portfolio_name = portfolio_data.get('name', f'Portfolio {idx + 1}')
            portfolio_stocks = portfolio_data['portfolio']
            
            tickers = [stock['ticker'] for stock in portfolio_stocks]
            weights = [stock['weight'] for stock in portfolio_stocks]
            portfolio_price_data = all_data[tickers]
            
            portfolio_returns = portfolio_price_data.pct_change().dot(weights)
            portfolio_cumulative = (1 + portfolio_returns).cumprod()
            
            metrics = calculate_portfolio_metrics(portfolio_price_data, weights)
            
            results[portfolio_name] = {
                'metrics': metrics,
                'cumulative_returns': portfolio_cumulative.tolist(),
                'weights': dict(zip(tickers, weights))
            }
        
        return {
            'success': True,
            'comparison_results': results,
            'dates': all_data.index.strftime('%Y-%m-%d').tolist()
        }
        
    except Exception as e:
        return {
            'success': False,
            'error': str(e)
        }

@app.route('/api/compare', methods=['POST'])
def compare_portfolios_endpoint():
    try:
        data = request.json
        portfolios = data['portfolios']
        start_date = data.get('startDate')
        
        results = compare_portfolios(portfolios, start_date)
        return jsonify(results)
        
    except Exception as e:
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

def calculate_correlation_matrix(portfolio_data):
    try:
        if portfolio_data.shape[1] == 1:
            ticker = portfolio_data.columns[0]
            return {
                'pearson': {
                    'matrix': {ticker: {ticker: 1.0}},
                    'min': 1.0,
                    'max': 1.0,
                    'mean': 1.0
                },
                'spearman': {
                    'matrix': {ticker: {ticker: 1.0}},
                    'min': 1.0,
                    'max': 1.0,
                    'mean': 1.0
                }
            }

        returns = portfolio_data.pct_change().dropna()
        
        pearson_corr = returns.corr(method='pearson')
        spearman_corr = returns.corr(method='spearman')
        
        correlations = {
            'pearson': {
                'matrix': pearson_corr.to_dict(),
                'min': float(pearson_corr.min().min()),
                'max': float(pearson_corr.max().max()),
                'mean': float(pearson_corr.mean().mean())
            },
            'spearman': {
                'matrix': spearman_corr.to_dict(),
                'min': float(spearman_corr.min().min()),
                'max': float(spearman_corr.max().max()),
                'mean': float(spearman_corr.mean().mean())
            }
        }
        
        return clean_for_json(correlations)
        
    except Exception as e:
        print(f"Error in calculate_correlation_matrix: {str(e)}")
        raise

@app.route('/api/correlation', methods=['POST'])
def get_correlation():
    try:
        data = request.json
        portfolio = data['portfolio']
        start_date = data.get('startDate')
        
        portfolio_data = pd.DataFrame()
        for stock in portfolio:
            stock_data = fetch_stock_data(stock['ticker'], start_date)
            if stock_data.empty:
                return jsonify({
                    'success': False,
                    'error': f'No data available for {stock["ticker"]}'
                }), 400
            portfolio_data[stock['ticker']] = stock_data['Adj Close']
        
        if portfolio_data.empty:
            return jsonify({
                'success': False,
                'error': 'No data available for correlation calculation'
            }), 400
        
        correlations = calculate_correlation_matrix(portfolio_data)
        
        return jsonify({
            'success': True,
            'correlations': correlations
        })
        
    except Exception as e:
        print(f"Error in get_correlation: {str(e)}")
        return jsonify({
            'success': False,
            'error': str(e)
        }), 400

if __name__ == '__main__':
    app.run(debug=True)