import os
import json
import logging
import traceback
from datetime import datetime
from flask import Flask, request, jsonify
from flask_cors import CORS
import yfinance as yf
import pandas as pd
import numpy as np
import empyrical as ep
import statsmodels.api as sm
from backend.routes.user_routes import user_bp 
import threading
import time
from backend.database.utils import (
    get_stock_data_from_db,
    save_stock_data_to_db,
    get_latest_date_for_ticker
)

app = Flask(__name__)
# Update CORS configuration to explicitly allow required headers
CORS(app, resources={
    r"/api/*": {
        "origins": ["http://localhost:5173"],
        "methods": ["GET", "POST", "DELETE", "OPTIONS"],
        "allow_headers": [
            "Content-Type",
            "Cache-Control",
            "Authorization"
        ]
    }
})

# Register the user blueprint
app.register_blueprint(user_bp, url_prefix='/api/user')

# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# Constants
PORTFOLIO_DIR = 'portfolios'
CACHE_DIR = 'cache'
if not os.path.exists(PORTFOLIO_DIR):
    os.makedirs(PORTFOLIO_DIR)
if not os.path.exists(CACHE_DIR):
    os.makedirs(CACHE_DIR)

# Utility Functions
def handle_preflight():
    """Handle CORS preflight requests"""
    response = app.make_default_options_response()
    response.headers['Access-Control-Allow-Methods'] = 'POST, OPTIONS'
    response.headers['Access-Control-Allow-Headers'] = 'Content-Type'
    return response

def get_cache_key(ticker, start_date):
    """Generate a cache key based on ticker and start date."""
    # Convert datetime to string format safe for filenames
    if isinstance(start_date, pd.Timestamp):
        start_date = start_date.strftime('%Y%m%d')
    elif isinstance(start_date, str):
        start_date = pd.to_datetime(start_date).strftime('%Y%m%d')
    return f"{ticker}_{start_date}.json"

def load_from_cache(cache_key):
    """Load data from cache if available."""
    try:
        cache_path = os.path.join(CACHE_DIR, cache_key)
        if os.path.exists(cache_path):
            data = pd.read_json(cache_path)
            # Convert string index back to datetime
            data.index = pd.to_datetime(data.index)
            if 'Adj Close' not in data.columns:
                logger.warning(f"Invalid cache data structure in {cache_key}")
                return None
            logger.info(f"Successfully loaded cache from {cache_path}")
            return data
    except Exception as e:
        logger.error(f"Failed to load cache for {cache_key}: {str(e)}")
    return None

def save_to_cache(cache_key, data):
    """Save data to cache."""
    try:
        cache_path = os.path.join(CACHE_DIR, cache_key)
        # Convert datetime index to string format before saving
        data.index = data.index.strftime('%Y-%m-%d')
        data.to_json(cache_path)
        logger.info(f"Successfully cached data to {cache_path}")
    except Exception as e:
        logger.error(f"Failed to save cache for {cache_key}: {str(e)}")

def clean_for_json(obj):
    """Clean objects to make them JSON serializable."""
    if isinstance(obj, (float, np.float32, np.float64)):
        if np.isnan(obj) or np.isinf(obj):
            return None  # Return null instead of 0 for NaN/Inf
        return float(obj)
    elif isinstance(obj, (int, np.int32, np.int64)):
        return int(obj)
    elif isinstance(obj, dict):
        return {k: clean_for_json(v) for k, v in obj.items()}
    elif isinstance(obj, (list, tuple)):
        return [clean_for_json(x) for x in obj]
    elif isinstance(obj, np.ndarray):
        return clean_for_json(obj.tolist())
    elif pd.isna(obj):
        return None  # Return null instead of 0 for pandas NA
    elif isinstance(obj, pd.Series):
        return clean_for_json(obj.tolist())
    elif isinstance(obj, pd.DataFrame):
        return clean_for_json(obj.to_dict('records'))
    elif obj is None:
        return None
    return obj

def fetch_stock_data(ticker, start_date):
    """
    Fetch historical stock data from database first, then yfinance if needed.
    Args:
        ticker (str): Stock ticker symbol.
        start_date (str): Start date in 'YYYY-MM-DD' format.
    Returns:
        pd.DataFrame: Historical stock data.
    """
    try:
        start_date = pd.to_datetime(start_date)
        ticker = ticker.strip().upper()
        
        # First, try to get data from database
        db_data = get_stock_data_from_db(ticker, start_date)
        
        if db_data is not None and not db_data.empty:
            latest_db_date = db_data.index.max()
            today = pd.Timestamp.now().normalize()
            
            # If data is up to date, return it
            if latest_db_date >= today - pd.Timedelta(days=1):
                logger.info(f"Using database data for {ticker}")
                return db_data
            
            # If we need to fetch new data
            new_start_date = latest_db_date + pd.Timedelta(days=1)
            logger.info(f"Fetching new data for {ticker} from {new_start_date}")
            
            try:
                # Fetch new data from yfinance
                new_data = yf.download(
                    ticker,
                    start=new_start_date,
                    progress=False,
                    auto_adjust=False,
                    timeout=30
                )
                
                if not new_data.empty:
                    # Save new data to database
                    save_stock_data_to_db(ticker, new_data)
                    
                    # Combine old and new data
                    combined_data = pd.concat([db_data, new_data])
                    return combined_data
                
                return db_data
                
            except Exception as e:
                logger.error(f"Failed to fetch new data for {ticker}: {str(e)}")
                return db_data  # Return existing data if new fetch fails
        
        # If no data in database, fetch all from yfinance
        logger.info(f"Fetching all data for {ticker} from yfinance")
        try:
            stock_data = yf.download(
                ticker,
                start=start_date,
                progress=False,
                auto_adjust=False,
                timeout=30
            )
            
            if not stock_data.empty:
                # Save to database
                save_stock_data_to_db(ticker, stock_data)
                return stock_data
            
            logger.warning(f"No data available for {ticker}")
            return None
            
        except Exception as e:
            logger.error(f"Failed to fetch data from yfinance for {ticker}: {str(e)}")
            return None

    except Exception as e:
        logger.error(f"Failed to fetch data for {ticker}: {str(e)}")
        return None

def calculate_portfolio_metrics(returns, benchmark_returns=None):
    """Calculate enhanced portfolio metrics."""
    try:
        metrics = {}
        logger.info("Starting portfolio metrics calculation...")
        
        # Ensure returns is a pandas Series and contains numeric values
        if isinstance(returns, pd.DataFrame):
            returns = returns.iloc[:, 0]  # Take first column if DataFrame
        
        # Convert returns to numeric, replacing any non-numeric values with NaN
        returns = pd.to_numeric(returns, errors='coerce').dropna()
        
        # Store returns for chart
        metrics['returns'] = returns.tolist()
        metrics['dates'] = returns.index.strftime('%Y-%m-%d').tolist()
        
        # Calculate cumulative returns
        cumulative_returns = (1 + returns).cumprod()
        metrics['cumulative_returns'] = cumulative_returns.tolist()
        
        # Basic portfolio metrics
        initial_value = 10000.0
        metrics['start_balance'] = initial_value
        metrics['end_balance'] = float(initial_value * cumulative_returns.iloc[-1])
        metrics['total_return'] = float((metrics['end_balance'] / initial_value) - 1)
        
        # Time period calculations
        total_days = len(returns)
        total_years = total_days / 252  # Assuming 252 trading days per year
        metrics['total_days'] = total_days
        metrics['total_years'] = total_years
        
        # Risk-free rate (2% annual)
        risk_free_rate = 0.02 / 252  # Daily risk-free rate
        
        try:
            # CAGR calculation
            metrics['cagr'] = float(((1 + metrics['total_return']) ** (1 / total_years)) - 1) if total_years > 0 else 0.0
            
            # Volatility calculations
            daily_std = float(returns.std())
            metrics['daily_volatility'] = daily_std
            metrics['annual_volatility'] = float(daily_std * np.sqrt(252))
            
            # Rolling volatility (30-day window)
            rolling_vol = returns.rolling(window=30).std() * np.sqrt(252)
            metrics['rolling_volatility'] = [float(x) if pd.notnull(x) else None for x in rolling_vol]
            
            # Risk metrics
            metrics['sharpe_ratio'] = float(ep.sharpe_ratio(returns, risk_free=risk_free_rate))
            metrics['sortino_ratio'] = float(ep.sortino_ratio(returns, required_return=risk_free_rate))
            metrics['max_drawdown'] = float(ep.max_drawdown(returns))
            
            # Calculate rolling returns with proper handling of NaN values
            rolling_returns = {}
            windows = {
                '1M': 21,
                '3M': 63,
                '6M': 126,
                '1Y': 252
            }
            
            for period, window in windows.items():
                if len(returns) >= window:
                    roll = returns.rolling(window=window).mean() * 252
                    rolling_returns[period] = [float(x) if pd.notnull(x) else None for x in roll]
                else:
                    rolling_returns[period] = [None] * len(returns)
            
            metrics['rolling_returns'] = rolling_returns
            
            # Value at Risk (VaR) calculations
            metrics['var_95'] = float(np.percentile(returns, 5))
            metrics['var_99'] = float(np.percentile(returns, 1))
            metrics['cvar_95'] = float(returns[returns <= metrics['var_95']].mean())
            
            # Return distribution metrics
            metrics['skewness'] = float(returns.skew())
            metrics['kurtosis'] = float(returns.kurtosis())
            
            logger.info(f"Base metrics calculated: CAGR={metrics['cagr']:.2%}, Vol={metrics['annual_volatility']:.2%}")
            
        except Exception as e:
            logger.error(f"Error calculating base metrics: {str(e)}")
            metrics.update({
                'cagr': 0.0,
                'annual_volatility': 0.0,
                'sharpe_ratio': 0.0,
                'sortino_ratio': 0.0,
                'max_drawdown': 0.0
            })

        # Benchmark comparison metrics
        if benchmark_returns is not None and isinstance(benchmark_returns, pd.Series):
            try:
                # Align returns for benchmark calculations
                common_dates = returns.index.intersection(benchmark_returns.index)
                aligned_returns = returns[common_dates]
                aligned_benchmark = benchmark_returns[common_dates]
                
                # Calculate excess returns
                excess_returns = aligned_returns - aligned_benchmark
                tracking_error = float(np.std(excess_returns) * np.sqrt(252))
                
                # Regression analysis
                X = sm.add_constant(aligned_benchmark)
                model = sm.OLS(aligned_returns, X).fit()
                
                metrics.update({
                    'alpha': float(model.params['const'] * 252),  # Annualized alpha
                    'beta': float(model.params[1]),
                    'r_squared': float(model.rsquared),
                    'tracking_error': tracking_error,
                    'information_ratio': float(excess_returns.mean() / np.std(excess_returns) * np.sqrt(252)),
                    'active_return': float((aligned_returns.mean() - aligned_benchmark.mean()) * 252),
                    'correlation': float(aligned_returns.corr(aligned_benchmark)),
                    'up_capture': float(
                        (aligned_returns[aligned_benchmark > 0] / aligned_benchmark[aligned_benchmark > 0]).mean()
                    ) if (aligned_benchmark > 0).any() else 1.0,
                    'down_capture': float(
                        (aligned_returns[aligned_benchmark < 0] / aligned_benchmark[aligned_benchmark < 0]).mean()
                    ) if (aligned_benchmark < 0).any() else 1.0
                })
                
                logger.info(f"Relative metrics calculated: Beta={metrics['beta']}, Alpha={metrics['alpha']}, "
                          f"Information Ratio={metrics['information_ratio']}")
                
            except Exception as e:
                logger.error(f"Error calculating benchmark metrics: {str(e)}")
                metrics.update({
                    'alpha': 0.0,
                    'beta': 1.0,
                    'r_squared': 0.0,
                    'tracking_error': 0.0,
                    'information_ratio': 0.0,
                    'active_return': 0.0,
                    'correlation': 0.0,
                    'up_capture': 1.0,
                    'down_capture': 1.0
                })

        return clean_for_json(metrics)
        
    except Exception as e:
        logger.error(f"Error in calculate_portfolio_metrics: {str(e)}")
        return {
            'error': str(e),
            'cagr': 0.0,
            'annual_volatility': 0.0,
            'sharpe_ratio': 0.0,
            'sortino_ratio': 0.0,
            'max_drawdown': 0.0,
            'beta': 1.0,
            'alpha': 0.0,
            'information_ratio': 0.0,
            'tracking_error': 0.0,
            'r_squared': 0.0
        }

def calculate_correlation_matrix(portfolio_data):
    """Calculate correlation matrices for the portfolio."""
    try:
        # Calculate returns
        returns = portfolio_data.pct_change().dropna()
        
        # Calculate Pearson correlation
        pearson_matrix = returns.corr(method='pearson')
        pearson_values = pearson_matrix.values[np.triu_indices_from(pearson_matrix.values, k=1)]
        
        # Calculate Spearman correlation
        spearman_matrix = returns.corr(method='spearman')
        spearman_values = spearman_matrix.values[np.triu_indices_from(spearman_matrix.values, k=1)]
        
        # Convert matrices to dictionaries
        pearson_dict = {ticker: pearson_matrix[ticker].to_dict() for ticker in pearson_matrix.index}
        spearman_dict = {ticker: spearman_matrix[ticker].to_dict() for ticker in spearman_matrix.index}
        
        # Calculate rolling correlations
        rolling_window = min(252, len(returns) // 2)  # Use 1 year or half the data length
        if rolling_window < 2:
            rolling_window = 2  # Minimum window size
        
        rolling_corr = returns.rolling(window=rolling_window).corr()
        
        # Get the most recent rolling correlations
        latest_rolling = rolling_corr.xs(returns.index[-1], level=0)
        rolling_dict = {ticker: latest_rolling[ticker].to_dict() for ticker in latest_rolling.index}
        
        return {
            'pearson': {
                'matrix': pearson_dict,
                'avg': float(np.mean(pearson_values)),
                'min': float(np.min(pearson_values)),
                'max': float(np.max(pearson_values))
            },
            'spearman': {
                'matrix': spearman_dict,
                'avg': float(np.mean(spearman_values)),
                'min': float(np.min(spearman_values)),
                'max': float(np.max(spearman_values))
            },
            'rolling': {
                'matrix': rolling_dict,
                'window': rolling_window
            }
        }
    except Exception as e:
        logger.error(f"Error in calculate_correlation_matrix: {str(e)}", exc_info=True)
        return None

def calculate_drawdown_periods(returns):
    """
    Identify drawdown periods in the portfolio.
    Args:
        returns (pd.Series): Daily returns of the portfolio.
    Returns:
        dict: Drawdown analysis including periods and chart data.
    """
    try:
        # Set a realistic initial portfolio value
        initial_value = 10000  # Start with $10,000
        cumulative = initial_value * (1 + returns).cumprod()
        running_max = cumulative.expanding().max()
        drawdown_series = (cumulative - running_max) / running_max
        
        # Ensure index is datetime
        returns.index = pd.to_datetime(returns.index)
        
        # Prepare chart data
        chart_data = {
            'dates': returns.index.strftime('%Y-%m-%d').tolist(),
            'drawdowns': [float(x) if pd.notnull(x) else 0 for x in drawdown_series.tolist()],
            'cumulative': [float(x) if pd.notnull(x) else initial_value for x in cumulative.tolist()],
            'peaks': [float(x) if pd.notnull(x) else initial_value for x in running_max.tolist()]
        }

        # Find significant drawdown periods (more than 5%)
        DRAWDOWN_THRESHOLD = -0.05  # 5% threshold
        significant_drawdowns = []
        in_drawdown = False
        peak_idx = None
        trough_idx = None

        for date in drawdown_series.index:
            current_drawdown = drawdown_series[date]
            
            # Skip NaN values
            if pd.isna(current_drawdown):
                continue
            
            # Start of drawdown
            if not in_drawdown and current_drawdown < DRAWDOWN_THRESHOLD:
                in_drawdown = True
                # Find the actual peak before this drawdown
                peak_idx = running_max[:date].idxmax()
                trough_idx = date
            
            # Update trough if in drawdown
            elif in_drawdown and current_drawdown < drawdown_series[trough_idx]:
                trough_idx = date
            
            # Check for recovery
            elif in_drawdown and cumulative[date] >= cumulative[peak_idx]:
                # Convert dates to string format
                peak_date = pd.to_datetime(peak_idx)
                trough_date = pd.to_datetime(trough_idx)
                recovery_date = pd.to_datetime(date)
                
                drawdown_info = {
                    'start': peak_date.strftime('%Y-%m-%d'),
                    'end': trough_date.strftime('%Y-%m-%d'),
                    'recovery': recovery_date.strftime('%Y-%m-%d'),
                    'depth': float(drawdown_series[trough_idx]),
                    'length_days': (trough_date - peak_date).days,
                    'recovery_time_days': (recovery_date - trough_date).days,
                    'peak_value': float(cumulative[peak_idx]),
                    'trough_value': float(cumulative[trough_idx]),
                    'recovery_value': float(cumulative[date])
                }
                significant_drawdowns.append(drawdown_info)
                
                # Reset for next drawdown
                in_drawdown = False
                peak_idx = None
                trough_idx = None

        # Handle ongoing drawdown
        if in_drawdown and peak_idx is not None and trough_idx is not None:
            peak_date = pd.to_datetime(peak_idx)
            trough_date = pd.to_datetime(trough_idx)
            
            drawdown_info = {
                'start': peak_date.strftime('%Y-%m-%d'),
                'end': trough_date.strftime('%Y-%m-%d'),
                'recovery': None,
                'depth': float(drawdown_series[trough_idx]),
                'length_days': (trough_date - peak_date).days,
                'recovery_time_days': None,
                'peak_value': float(cumulative[peak_idx]),
                'trough_value': float(cumulative[trough_idx]),
                'recovery_value': None,
                'is_active': True
            }
            significant_drawdowns.append(drawdown_info)

        # Sort drawdowns by depth (most severe first)
        significant_drawdowns.sort(key=lambda x: x['depth'])
        max_drawdowns_returned = significant_drawdowns[:10]  # Return top 10 worst drawdowns

        logger.info(f"Found {len(max_drawdowns_returned)} significant drawdown periods")
        return {
            'periods': max_drawdowns_returned,
            'chart_data': chart_data
        }

    except Exception as e:
        logger.error(f"Error in calculate_drawdown_periods: {str(e)}", exc_info=True)
        return {
            'periods': [],
            'chart_data': {
                'dates': [],
                'drawdowns': [],
                'cumulative': [],
                'peaks': []
            }
        }

def calculate_rebalancing_impact(portfolio_data, weights, transaction_cost):
    """Calculate the impact of different rebalancing strategies."""
    try:
        logger.info("Starting rebalancing impact analysis...")
        
        # Ensure weights are numpy array and sum to 1
        weights = np.array(weights)
        if not np.isclose(weights.sum(), 1.0):
            weights = weights / weights.sum()
            
        # Calculate returns
        returns = portfolio_data.pct_change().fillna(0)
        
        # Define rebalancing strategies
        strategies = {
            'Buy and Hold': None,  # No rebalancing
            'Monthly': 'M',
            'Quarterly': 'Q',
            'Semi-Annual': '6M',
            'Annual': 'Y'
        }
        
        results = {}
        initial_value = 10000.0  # Start with $10,000
        
        for strategy_name, freq in strategies.items():
            try:
                # Initialize portfolio tracking
                portfolio_value = initial_value
                current_weights = weights.copy()
                current_holdings = portfolio_value * current_weights
                total_costs = 0
                rebalance_count = 0
                portfolio_values = []
                daily_returns = []
                turnover = []
                tracking_error = []
                current_weights_history = []
                
                last_rebalance = None
                
                # Track daily portfolio changes
                for date in returns.index:
                    # Update holdings based on daily returns
                    daily_return_vector = returns.loc[date].values
                    current_holdings *= (1 + daily_return_vector)
                    portfolio_value = current_holdings.sum()
                    
                    # Calculate daily portfolio return
                    if portfolio_values:
                        daily_return = (portfolio_value / portfolio_values[-1]) - 1
                    else:
                        daily_return = 0
                        
                    # Update current weights
                    current_weights = current_holdings / portfolio_value
                    
                    # Check if rebalancing is needed
                    should_rebalance = False
                    if freq is not None:  # Skip for buy and hold
                        current_period = pd.Timestamp(date).to_period(freq)
                        last_period = pd.Timestamp(last_rebalance).to_period(freq) if last_rebalance is not None else None
                        
                        if last_period is None or current_period != last_period:
                            # Calculate weight drift
                            weight_drift = np.abs(current_weights - weights).sum() / 2
                            
                            # Only rebalance if drift exceeds threshold
                            if weight_drift > 0.01:  # 1% threshold
                                should_rebalance = True
                    
                    if should_rebalance:
                        # Calculate trades needed
                        target_holdings = portfolio_value * weights
                        trades = target_holdings - current_holdings
                        
                        # Calculate transaction costs
                        trade_costs = abs(trades).sum() * transaction_cost
                        
                        # Apply transaction costs
                        portfolio_value -= trade_costs
                        total_costs += trade_costs
                        
                        # Update holdings and record rebalancing
                        current_holdings = portfolio_value * weights
                        last_rebalance = date
                        rebalance_count += 1
                        
                        # Calculate turnover
                        turnover.append(abs(trades).sum() / (2 * portfolio_value))
                        
                        # Adjust daily return for transaction costs
                        daily_return -= trade_costs / portfolio_values[-1] if portfolio_values else 0
                    
                    # Record tracking data
                    portfolio_values.append(portfolio_value)
                    daily_returns.append(daily_return)
                    current_weights_history.append(current_weights.tolist())
                    
                    # Calculate tracking error vs target weights
                    weight_error = np.sqrt(np.sum((current_weights - weights) ** 2))
                    tracking_error.append(weight_error)
                
                # Convert to numpy array for calculations
                daily_returns = np.array(daily_returns)
                
                # Calculate strategy metrics
                metrics = calculate_portfolio_metrics(pd.Series(daily_returns, index=returns.index))
                
                # Add rebalancing-specific metrics
                metrics.update({
                    'portfolio_values': portfolio_values,
                    'final_value': portfolio_values[-1],
                    'total_return': (portfolio_values[-1] / initial_value) - 1,
                    'rebalance_count': rebalance_count,
                    'total_costs': total_costs,
                    'cost_percentage': total_costs / portfolio_values[-1],
                    'avg_turnover': np.mean(turnover) if turnover else 0,
                    'avg_tracking_error': np.mean(tracking_error),
                    'weight_drift': {
                        'mean': float(np.mean(tracking_error)),
                        'max': float(np.max(tracking_error)),
                        'current': float(tracking_error[-1])
                    },
                    'current_weights': current_weights.tolist(),
                    'weight_history': current_weights_history
                })
                
                results[strategy_name] = clean_for_json(metrics)
                
            except Exception as e:
                logger.error(f"Error calculating {strategy_name} strategy: {str(e)}")
                results[strategy_name] = {
                    'error': str(e),
                    'final_value': initial_value,
                    'total_return': 0.0,
                    'rebalance_count': 0,
                    'total_costs': 0.0
                }
        
        return results
            
    except Exception as e:
        logger.error(f"Error in calculate_rebalancing_impact: {str(e)}")
        return {}

def calculate_attribution(portfolio_data, weights, benchmark_data):
    """Calculate enhanced performance attribution metrics."""
    try:
        logger.info("Starting attribution analysis...")
        
        # Calculate returns using Close prices
        portfolio_returns = portfolio_data.pct_change().fillna(0)
        benchmark_returns = benchmark_data['Close'].pct_change().fillna(0)
        
        # Calculate weighted portfolio returns
        weighted_returns = (portfolio_returns * weights).sum(axis=1)
        
        # Calculate cumulative returns
        portfolio_cumulative = (1 + weighted_returns).cumprod()
        benchmark_cumulative = (1 + benchmark_returns).cumprod()
        
        # Calculate risk-free rate (assuming 2% annual)
        risk_free_rate = 0.02 / 252  # Daily risk-free rate
        
        try:
            # Factor Attribution using statsmodels
            X = sm.add_constant(benchmark_returns)
            model = sm.OLS(weighted_returns, X).fit()
            alpha = model.params['const']
            beta = model.params[1]
            r_squared = float(model.rsquared)
            
            # Calculate rolling betas and alphas
            rolling_window = min(252, len(weighted_returns) // 4)  # Use 1 year or quarter of data
            rolling_betas = []
            rolling_alphas = []
            rolling_r2 = []
            
            for i in range(rolling_window, len(weighted_returns)):
                window_returns = weighted_returns[i-rolling_window:i]
                window_benchmark = benchmark_returns[i-rolling_window:i]
                X_window = sm.add_constant(window_benchmark)
                window_model = sm.OLS(window_returns, X_window).fit()
                rolling_betas.append(float(window_model.params[1]))
                rolling_alphas.append(float(window_model.params[0] * 252))  # Annualized
                rolling_r2.append(float(window_model.rsquared))
            
            # Performance Attribution
            # Market Component
            market_return = benchmark_returns.mean() * 252  # Annualized
            market_contribution = beta * market_return
            
            # Security Selection (Alpha)
            security_selection = alpha * 252  # Annualized
            
            # Risk-Adjusted Performance
            excess_returns = weighted_returns - benchmark_returns
            tracking_error = float(np.std(excess_returns) * np.sqrt(252))
            information_ratio = float(excess_returns.mean() / np.std(excess_returns) * np.sqrt(252))
            
            # Capture Ratios
            up_periods = benchmark_returns > 0
            down_periods = benchmark_returns < 0
            
            up_capture = float((weighted_returns[up_periods] / benchmark_returns[up_periods]).mean()) if up_periods.any() else 1.0
            down_capture = float((weighted_returns[down_periods] / benchmark_returns[down_periods]).mean()) if down_periods.any() else 1.0
            
            # Calculate rolling correlations
            rolling_corr = weighted_returns.rolling(window=rolling_window).corr(benchmark_returns)
            
            # Active Risk Analysis
            active_risk = np.std(excess_returns) * np.sqrt(252)
            active_return = (weighted_returns.mean() - benchmark_returns.mean()) * 252
            
            # Risk-Adjusted Metrics
            treynor_ratio = (weighted_returns.mean() * 252 - risk_free_rate * 252) / beta if beta != 0 else 0
            
            # Factor Attribution Results
            factor_attribution = {
                'alpha': float(alpha * 252),  # Annualized
                'beta': float(beta),
                'r_squared': float(r_squared),
                'tracking_error': float(tracking_error),
                'information_ratio': float(information_ratio),
                'up_capture': float(up_capture),
                'down_capture': float(down_capture),
                'active_risk': float(active_risk),
                'active_return': float(active_return),
                'treynor_ratio': float(treynor_ratio),
                'market_contribution': float(market_contribution),
                'security_selection': float(security_selection),
                'rolling_metrics': {
                    'betas': rolling_betas,
                    'alphas': rolling_alphas,
                    'r_squared': rolling_r2,
                    'correlation': rolling_corr.tolist()
                }
            }
            
            # Performance Decomposition
            total_attribution = {
                'market': float(market_contribution),
                'selection': float(security_selection),
                'interaction': float(active_return - market_contribution - security_selection)
            }
            
            # Stock-Level Attribution
            stock_attribution = {}
            for ticker, weight in zip(portfolio_data.columns, weights):
                stock_returns = portfolio_returns[ticker]
                stock_contribution = float(stock_returns.mean() * 252 * weight)  # Annualized contribution
                
                # Calculate individual stock metrics
                stock_excess = stock_returns - benchmark_returns
                stock_tracking_error = float(np.std(stock_excess) * np.sqrt(252))
                stock_info_ratio = float(stock_excess.mean() / np.std(stock_excess) * np.sqrt(252)) if stock_tracking_error > 0 else 0
                
                # Get stock sector info from database
                stock_sector = ''  # You can get this from your database
                
                stock_attribution[ticker] = {
                    'weight': float(weight),
                    'contribution': stock_contribution,
                    'sector': stock_sector,
                    'tracking_error': stock_tracking_error,
                    'information_ratio': stock_info_ratio,
                    'correlation_with_benchmark': float(stock_returns.corr(benchmark_returns)),
                    'beta': float(np.cov(stock_returns, benchmark_returns)[0,1] / np.var(benchmark_returns))
                }
            
            return {
                'factor': factor_attribution,
                'total': total_attribution,
                'stocks': stock_attribution,
                'cumulative_returns': {
                    'portfolio': portfolio_cumulative.tolist(),
                    'benchmark': benchmark_cumulative.tolist(),
                    'dates': portfolio_returns.index.strftime('%Y-%m-%d').tolist()
                }
            }
            
        except Exception as e:
            logger.error(f"Error in attribution calculations: {str(e)}")
            return None
            
    except Exception as e:
        logger.error(f"Error in calculate_attribution: {str(e)}")
        return None

# API Routes
@app.route('/api/compare', methods=['POST', 'OPTIONS'])
def compare_portfolios():
    global analysis_progress
    
    if request.method == 'OPTIONS':
        return handle_preflight()

    try:
        logger.info("Starting portfolio comparison...")
        analysis_progress['progress'] = 0
        analysis_progress['status'] = 'processing'
        analysis_progress['error'] = None

        data = request.get_json()
        logger.info(f"Received request data: {json.dumps(data, indent=2)}")
        
        portfolios = data.get('portfolios', [])
        start_date = data.get('startDate')
        transaction_cost = min(float(data.get('transactionCost', 0.001)), 0.01)

        if not portfolios:
            update_progress(0, 'failed', 'No portfolios provided for comparison')
            return jsonify({
                'success': False,
                'error': 'No portfolios provided for comparison'
            }), 400

        comparison_results = {}
        start_date = pd.to_datetime(start_date)
        all_data = pd.DataFrame()
        
        # Debug log for portfolios
        logger.info(f"Processing portfolios: {json.dumps([p['name'] for p in portfolios])}")
        
        # Fetch data for all tickers
        unique_tickers = set()
        for portfolio in portfolios:
            portfolio_stocks = portfolio.get('portfolio', [])
            unique_tickers.update([stock['ticker'] for stock in portfolio_stocks])
        logger.info(f"Fetching data for tickers: {list(unique_tickers)}")
        
        update_progress(10)
        total_tickers = len(unique_tickers)
        for i, ticker in enumerate(unique_tickers, 1):
            stock_data = fetch_stock_data(ticker, start_date)
            if stock_data is not None and not stock_data.empty:
                all_data[ticker] = stock_data['Close']
                logger.info(f"Successfully fetched data for {ticker}, shape: {stock_data.shape}")
            else:
                logger.error(f"Failed to fetch data for {ticker}")
                update_progress(0, 'failed', f'Failed to fetch data for {ticker}')
                return jsonify({
                    'success': False,
                    'error': f'Failed to fetch data for {ticker}'
                }), 400
            progress = 10 + (30 * i / total_tickers)
            update_progress(int(progress))

        all_data = all_data.dropna()
        logger.info(f"All data shape after dropna: {all_data.shape}")

        if len(all_data) < 252:
            update_progress(0, 'failed', 'Insufficient historical data')
            return jsonify({
                'success': False,
                'error': 'Insufficient historical data. Need at least 252 data points.'
            }), 400

        update_progress(50)

        # Calculate metrics for each portfolio
        total_portfolios = len(portfolios)
        for i, portfolio in enumerate(portfolios, 1):
            portfolio_name = portfolio.get('name', 'Unnamed Portfolio')
            portfolio_stocks = portfolio.get('portfolio', [])
            logger.info(f"Processing portfolio: {portfolio_name}")
            
            # Calculate portfolio weights
            weights = np.array([float(stock['weight']) / 100 for stock in portfolio_stocks])
            if not np.isclose(weights.sum(), 1.0):
                weights = weights / weights.sum()
            
            tickers = [stock['ticker'] for stock in portfolio_stocks]
            portfolio_data = all_data[tickers]
            portfolio_returns = portfolio_data.pct_change().fillna(0)
            weighted_returns = (portfolio_returns * weights).sum(axis=1)
            
            # Debug log for returns
            logger.info(f"Portfolio {portfolio_name} returns stats: mean={weighted_returns.mean():.4f}, std={weighted_returns.std():.4f}")
            
            # Get benchmark returns if this is not the benchmark portfolio
            benchmark_returns = None
            if portfolio_name != 'SPY' and 'SPY' in all_data.columns:
                benchmark_returns = all_data['SPY'].pct_change().fillna(0)
                logger.info(f"Using SPY as benchmark for {portfolio_name}")
            
            # Calculate metrics
            metrics = calculate_portfolio_metrics(weighted_returns, benchmark_returns)
            logger.info(f"Calculated metrics for {portfolio_name}: {json.dumps(metrics, indent=2, default=str)}")
            
            # Calculate drawdowns
            drawdowns = calculate_drawdown_periods(weighted_returns)
            metrics['drawdowns'] = drawdowns
            
            # Calculate rebalancing impact
            rebalancing = calculate_rebalancing_impact(portfolio_data, weights, transaction_cost)
            
            # Calculate attribution if benchmark exists
            attribution = None
            if benchmark_returns is not None:
                attribution = calculate_attribution(
                    portfolio_data,
                    weights,
                    pd.DataFrame({'Close': all_data['SPY']})
                )
            
            # Calculate correlations for portfolios with multiple stocks
            correlations = None
            if len(tickers) > 1:
                correlations = calculate_correlation_matrix(portfolio_data)
            
            # Debug log before adding to results
            logger.info(f"Portfolio {portfolio_name} metrics calculated successfully")
            
            comparison_results[portfolio_name] = {
                'metrics': metrics,
                'rebalancing': rebalancing,
                'portfolio': portfolio_stocks,
                'attribution': attribution,
                'correlations': correlations
            }
            
            # Debug log for results
            logger.info(f"Added results for {portfolio_name}")
            
            progress = 50 + (40 * i / total_portfolios)
            update_progress(int(progress))

        update_progress(100, 'completed')

        # Final debug log for response
        response_data = {
            'success': True,
            'comparison_results': comparison_results,
            'data_quality': {
                'num_data_points': len(all_data),
                'date_range': {
                    'start': all_data.index[0].strftime('%Y-%m-%d'),
                    'end': all_data.index[-1].strftime('%Y-%m-%d')
                },
                'mode': 'benchmark' if any(p['name'] == 'SPY' for p in portfolios) else 'portfolio_comparison'
            }
        }
        
        # Test JSON serialization before returning
        try:
            json_str = json.dumps(response_data, default=str)
            logger.info("Successfully serialized response to JSON")
            return jsonify(response_data)
        except Exception as e:
            logger.error(f"JSON serialization failed: {str(e)}")
            # Try to identify problematic values
            import numpy as np
            def find_nan(obj, path=""):
                if isinstance(obj, dict):
                    for k, v in obj.items():
                        yield from find_nan(v, f"{path}.{k}" if path else k)
                elif isinstance(obj, (list, tuple)):
                    for i, v in enumerate(obj):
                        yield from find_nan(v, f"{path}[{i}]")
                elif isinstance(obj, float) and (np.isnan(obj) or np.isinf(obj)):
                    yield path, obj
            
            nan_values = list(find_nan(response_data))
            logger.error(f"Found NaN/Inf values at: {nan_values}")
            
            # Clean the response data
            response_data = clean_for_json(response_data)
            return jsonify(response_data)

    except Exception as e:
        logger.error(f"Error in compare_portfolios: {str(e)}")
        analysis_progress['status'] = 'failed'
        analysis_progress['error'] = str(e)
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500

@app.route('/api/backtest', methods=['POST'])
def backtest():
    """
    Backtest the portfolio against historical data.
    Expects JSON payload with:
        - portfolio: list of stocks with 'ticker' and 'weight'
        - startDate: backtest start date in 'YYYY-MM-DD'
    """
    try:
        logger.info("Starting backtest...")
        data = request.get_json()
        logger.info(f"Received backtest data: {json.dumps(data, indent=2)}")
        
        portfolio = data.get('portfolio', [])
        start_date = data.get('startDate')
        
        if not portfolio:
            return jsonify({
                'success': False,
                'error': 'No portfolio provided for backtest'
            }), 400

        # Validate portfolio
        is_valid, error_msg = validate_portfolio(portfolio)
        if not is_valid:
            return jsonify({
                'success': False,
                'error': error_msg
            }), 400

        start_date = pd.to_datetime(start_date)
        portfolio_data = pd.DataFrame()

        # Fetch data for portfolio stocks
        logger.info("Fetching portfolio data...")
        for stock in portfolio:
            ticker = stock['ticker']
            stock_data = fetch_stock_data(ticker, start_date)
            if stock_data is not None and not stock_data.empty:
                portfolio_data[ticker] = stock_data['Adj Close']
                logger.info(f"Data fetched for {ticker}, shape: {stock_data.shape}")
            else:
                return jsonify({
                    'success': False,
                    'error': f'No data available for {ticker}'
                }), 400

        # Fetch or define benchmark
        benchmark = 'SPY'  # Default benchmark
        benchmark_data = fetch_stock_data(benchmark, start_date)
        if benchmark_data is None or benchmark_data.empty:
            return jsonify({
                'success': False,
                'error': f'No data available for benchmark {benchmark}'
            }), 400

        # Calculate metrics
        portfolio_returns = portfolio_data.pct_change().dropna()
        benchmark_returns = benchmark_data['Adj Close'].pct_change().dropna().reindex(portfolio_returns.index)
        benchmark_returns = benchmark_returns.fillna(0)

        portfolio_metrics = calculate_portfolio_metrics(portfolio_returns, benchmark_returns)
        benchmark_metrics = calculate_portfolio_metrics(benchmark_returns)

        # Calculate cumulative portfolio and benchmark values
        initial_investment = portfolio_metrics.get('start_balance', 10000.0)
        portfolio_cumulative = (1 + portfolio_returns).cumprod() * initial_investment
        benchmark_cumulative = (1 + benchmark_returns).cumprod() * initial_investment

        # Calculate drawdowns
        portfolio_drawdowns = calculate_drawdown_periods(portfolio_returns)
        benchmark_drawdowns = calculate_drawdown_periods(benchmark_returns)

        # Prepare response with data quality information
        common_dates = portfolio_returns.index
        response_data = {
            'success': True,
            'portfolio_metrics': clean_for_json(portfolio_metrics),
            'benchmark_metrics': clean_for_json(benchmark_metrics),
            'historical_performance': {
                'dates': common_dates.strftime('%Y-%m-%d').tolist(),
                'portfolio_values': portfolio_cumulative.tolist(),
                'benchmark_values': benchmark_cumulative.tolist(),
                'initial_investment': initial_investment
            },
            'drawdowns': {
                'portfolio_drawdowns': portfolio_drawdowns,
                'benchmark_drawdowns': benchmark_drawdowns
            },
            'data_quality': {
                'num_data_points': len(common_dates),
                'date_range': {
                    'start': common_dates[0].strftime('%Y-%m-%d'),
                    'end': common_dates[-1].strftime('%Y-%m-%d')
                },
                'missing_values': False,
                'aligned_data': True
            }
        }

        logger.info("Backtest completed successfully")
        return jsonify(response_data)

    except Exception as e:
        logger.error(f"Error in backtest: {str(e)}", exc_info=True)
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500

@app.route('/api/correlation', methods=['POST'])
def get_correlation():
    """
    Calculate and return correlation matrices for the portfolio.
    Expects JSON payload with:
        - portfolio: list of stocks with 'ticker' and 'weight'
    """
    try:
        logger.info("Starting correlation analysis...")
        data = request.get_json()
        logger.info(f"Received correlation data: {json.dumps(data, indent=2)}")
        
        portfolio = data.get('portfolio', [])
        if not portfolio or len(portfolio) < 2:
            return jsonify({
                'success': False,
                'error': 'Portfolio must contain at least two stocks for correlation analysis'
            }), 400

        # Validate portfolio
        is_valid, error_msg = validate_portfolio(portfolio)
        if not is_valid:
            return jsonify({
                'success': False,
                'error': f"Invalid portfolio: {error_msg}"
            }), 400

        # Fetch data for portfolio stocks
        portfolio_tickers = [stock['ticker'] for stock in portfolio]
        portfolio_data = pd.DataFrame()

        logger.info("Fetching portfolio data for correlation analysis...")
        for ticker in portfolio_tickers:
            stock_data = fetch_stock_data(ticker, start_date='2000-01-01')  # Using a fixed start date
            if stock_data is not None and not stock_data.empty:
                portfolio_data[ticker] = stock_data['Adj Close']
                logger.info(f"Data fetched for {ticker}, shape: {stock_data.shape}")
            else:
                return jsonify({
                    'success': False,
                    'error': f'No data available for {ticker}'
                }), 400

        # Calculate correlation matrices
        correlations = calculate_correlation_matrix(portfolio_data)

        return jsonify({
            'success': True,
            'correlations': correlations
        })

    except Exception as e:
        logger.error(f"Error in get_correlation: {str(e)}", exc_info=True)
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500

@app.route('/api/attribution', methods=['POST', 'OPTIONS'])
def get_attribution():
    """Get portfolio attribution analysis."""
    if request.method == 'OPTIONS':
        response = app.make_default_options_response()
        response.headers['Access-Control-Allow-Methods'] = 'POST'
        response.headers['Access-Control-Allow-Headers'] = 'Content-Type'
        return response

    try:
        data = request.get_json()
        portfolio = data.get('portfolio', [])
        start_date = data.get('startDate')
        benchmark = data.get('benchmark', 'SPY')
        
        if not portfolio:
            return jsonify({
                'success': False,
                'error': 'No portfolio provided'
            }), 400

        # Validate portfolio
        is_valid, error_msg = validate_portfolio(portfolio)
        if not is_valid:
            return jsonify({
                'success': False,
                'error': f"Invalid portfolio: {error_msg}"
            }), 400

        start_date = pd.to_datetime(start_date)
        portfolio_tickers = [stock['ticker'] for stock in portfolio]
        portfolio_weights = np.array([float(stock['weight']) for stock in portfolio])

        # Fetch data for portfolio stocks
        logger.info("Fetching portfolio data for attribution analysis...")
        portfolio_data = pd.DataFrame()
        for ticker in portfolio_tickers:
            stock_data = fetch_stock_data(ticker, start_date)
            if stock_data is not None and not stock_data.empty:
                portfolio_data[ticker] = stock_data['Adj Close']
                logger.info(f"Data fetched for {ticker}, shape: {stock_data.shape}")
            else:
                return jsonify({
                    'success': False,
                    'error': f'No data available for {ticker}'
                }), 400

        # Fetch benchmark data
        logger.info(f"Fetching benchmark data for {benchmark}")
        benchmark_data = fetch_stock_data(benchmark, start_date)
        if benchmark_data is None or benchmark_data.empty:
            return jsonify({
                'success': False,
                'error': f'No data available for benchmark {benchmark}'
            }), 400

        # Calculate attribution
        attribution_results = calculate_attribution(portfolio_data, portfolio_weights, benchmark_data)
        if attribution_results is None:
            return jsonify({
                'success': False,
                'error': 'Failed to calculate attribution'
            }), 500

        return jsonify({
            'success': True,
            'attribution': attribution_results
        })

    except Exception as e:
        logger.error(f"Error in get_attribution: {str(e)}", exc_info=True)
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500

def validate_portfolio(portfolio):
    """Validate portfolio structure and weights."""
    try:
        if not portfolio or not isinstance(portfolio, list):
            return False, "Invalid portfolio format"

        # Validate each position
        for position in portfolio:
            if not isinstance(position, dict):
                return False, "Invalid position format"
            if 'ticker' not in position or 'weight' not in position:
                return False, "Missing ticker or weight"
            
            # Convert weight to float if it's a string
            if isinstance(position['weight'], str):
                position['weight'] = float(position['weight'])

        # Calculate total weight
        total_weight = sum(float(position['weight']) for position in portfolio)
        
        # Check if weights are in percentage format (0-100) or decimal format (0-1)
        is_percentage = total_weight > 1.1  # Assume percentage if sum > 1.1
        
        if is_percentage:
            # Convert to decimal if in percentage format
            for position in portfolio:
                position['weight'] = float(position['weight']) / 100
        
        # Validate total weight is close to 1.0 after normalization
        total_weight = sum(float(position['weight']) for position in portfolio)
        if not np.isclose(total_weight, 1.0, rtol=1e-3):
            # Normalize weights to sum to 1.0
            for position in portfolio:
                position['weight'] = float(position['weight']) / total_weight

        return True, ""
        
    except Exception as e:
        return False, f"Validation error: {str(e)}"

# Portfolio management routes
@app.route('/api/portfolio/load', methods=['GET'])
def load_portfolios():
    """Load all saved portfolios"""
    try:
        portfolios = []
        if os.path.exists(PORTFOLIO_DIR):
            for filename in os.listdir(PORTFOLIO_DIR):
                if filename.endswith('.json'):
                    file_path = os.path.join(PORTFOLIO_DIR, filename)
                    try:
                        with open(file_path, 'r') as f:
                            portfolio = json.load(f)
                            portfolio['filename'] = filename
                            # Add creation date from file metadata
                            created_at = os.path.getctime(file_path)
                            portfolio['created_at'] = datetime.fromtimestamp(created_at).isoformat()
                            portfolios.append(portfolio)
                    except Exception as e:
                        logger.error(f"Error loading portfolio {filename}: {str(e)}")
                        continue

        # Sort portfolios by creation date, newest first
        portfolios.sort(key=lambda x: x.get('created_at', ''), reverse=True)
        
        return jsonify({
            'success': True,
            'portfolios': portfolios
        })
    except Exception as e:
        logger.error(f"Error in load_portfolios: {str(e)}")
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500

@app.route('/api/portfolio/save', methods=['POST'])
def save_portfolio():
    """Save a portfolio"""
    try:
        data = request.get_json()
        if not data:
            return jsonify({
                'success': False,
                'error': 'No data provided'
            }), 400

        name = data.get('name')
        portfolio = data.get('portfolio', [])

        if not name:
            return jsonify({
                'success': False,
                'error': 'Portfolio name is required'
            }), 400

        # Log received portfolio data for debugging
        logger.info(f"Received portfolio data: {json.dumps(portfolio, indent=2)}")

        # Validate portfolio structure and weights
        is_valid, error_msg = validate_portfolio(portfolio)
        if not is_valid:
            return jsonify({
                'success': False,
                'error': error_msg
            }), 400

        # Create filename from name and timestamp
        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
        safe_name = "".join(x for x in name if x.isalnum() or x in (' ', '-', '_')).strip().replace(' ', '_')
        filename = f"portfolio_{timestamp}_{safe_name}.json"
        file_path = os.path.join(PORTFOLIO_DIR, filename)

        # Save portfolio with weights in decimal form
        with open(file_path, 'w') as f:
            json.dump({
                'name': name,
                'portfolio': portfolio  # Weights should already be in decimal form
            }, f, indent=2)

        logger.info(f"Portfolio saved successfully: {filename}")
        return jsonify({
            'success': True,
            'filename': filename
        })
    except Exception as e:
        logger.error(f"Error in save_portfolio: {str(e)}")
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500

@app.route('/api/portfolio/delete/<path:filename>', methods=['DELETE', 'OPTIONS'])
def delete_portfolio_by_name(filename):
    """Delete a portfolio"""
    # Handle preflight request
    if request.method == 'OPTIONS':
        response = app.make_default_options_response()
        response.headers['Access-Control-Allow-Methods'] = 'DELETE'
        response.headers['Access-Control-Allow-Headers'] = 'Content-Type'
        return response

    try:
        if not filename.endswith('.json'):
            return jsonify({
                'success': False,
                'error': 'Invalid filename'
            }), 400

        file_path = os.path.join(PORTFOLIO_DIR, filename)
        if not os.path.exists(file_path):
            return jsonify({
                'success': False,
                'error': 'Portfolio not found'
            }), 404

        try:
            # Delete the file
            os.remove(file_path)
            logger.info(f"Successfully deleted portfolio: {filename}")
        except Exception as e:
            logger.error(f"Error deleting file {filename}: {str(e)}")
            return jsonify({
                'success': False,
                'error': f'Failed to delete file: {str(e)}'
            }), 500

        # Return updated portfolio list
        return load_portfolios()
    except Exception as e:
        logger.error(f"Error in delete_portfolio: {str(e)}")
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500

@app.after_request
def after_request(response):
    """Add CORS headers to all responses"""
    response.headers.add('Access-Control-Allow-Origin', 'http://localhost:5173')
    response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Cache-Control,Authorization')
    response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS')
    return response

@app.route('/api/health', methods=['HEAD', 'GET'])
def health_check():
    """Simple health check endpoint"""
    return '', 200

# Add this global variable to track progress
analysis_progress = {
    'progress': 0,
    'status': 'idle',  # 'idle', 'processing', 'completed', 'failed'
    'error': None
}

# Add this function to update progress
def update_progress(value, status='processing', error=None):
    global analysis_progress
    analysis_progress['progress'] = value
    analysis_progress['status'] = status
    analysis_progress['error'] = error

# Add this new endpoint
@app.route('/api/progress', methods=['GET'])
def get_progress():
    """Get the current analysis progress."""
    global analysis_progress
    return jsonify(analysis_progress)

if __name__ == '__main__':
    app.run(debug=True)

Analysis error: Error: local variable 'np' referenced before assignment
    at Object.compare (api.js:206:15)
    at async runAnalysis (App.jsx:428:26)
runAnalysis @ App.jsx:466