import asyncio
import logging
import os
from typing import Dict, List, Tuple, Optional, Any
from dataclasses import dataclass
from enum import Enum
import numpy as np
import pandas as pd
from datetime import datetime, timedelta
from collections import defaultdict
import time
import warnings
from cachetools import TTLCache
# Django imports
import django
from django.utils import timezone
from asgiref.sync import sync_to_async
from django.db.models import Q

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'binancebot.settings')
django.setup()

from .models import Kline, OrderBook, Signal, TradingPair
recent_snapshots_cache = TTLCache(maxsize=100, ttl=60)
# Enhanced logging configuration
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s - %(funcName)s:%(lineno)d',
    handlers=[
        logging.FileHandler('trading_signals.log'),
        logging.StreamHandler()
    ]
)
logger = logging.getLogger(__name__)

class SignalType(Enum):
    STRONG_BUY = "Strong Buy"
    BUY = "Buy"
    NEUTRAL = "Neutral"
    SELL = "Sell"
    STRONG_SELL = "Strong Sell"
    REVERSAL_BULLISH = "Reversal Bullish"
    REVERSAL_BEARISH = "Reversal Bearish"
    NO_SIGNAL = "No Signal"

@dataclass
class MarketRegime:
    volatility: float
    trend_strength: float
    liquidity_score: float
    regime_type: str
    confidence: float
    volume_profile: Dict[str, float]
    market_efficiency: float
    regime_change_probability: float

@dataclass
class MarketImpact:
    price_impact: float
    volume_impact: float
    slippage_estimate: float
    recovery_time: float
    impact_asymmetry: float
    market_resilience: float
    depth_score: float

@dataclass
class SignalMetrics:
    signal_time: datetime
    chart_timestamp: datetime
    symbol: str
    current_price: float
    price_24h_change: float
    signal_type: SignalType
    stop_loss: float
    take_profit: float
    success_rate: float
    result: str
    volume_analysis: Dict[str, Any]
    orderbook_analysis: Dict[str, Any]
    technical_analysis: Dict[str, Any]
    market_regime: MarketRegime
    market_impact: MarketImpact
    reversal_analysis: Optional[Dict[str, Any]] = None
    institutional_analysis: Optional[Dict[str, Any]] = None
    delta_volume_analysis: Optional[Dict[str, Any]] = None
    vwap_analysis: Optional[Dict[str, Any]] = None
    signal_confidence: float = 0.0
    false_signal_probability: float = 0.0
    contributing_factors: List[str] = None  # New field to record contributing analyses
    analysis_type: str = None  # New field to record the analysis type


class DynamicThresholds:
    """Dynamic thresholds that adapt to changing market conditions"""

    def __init__(self, window_size: int = 100):
        self.window_size = window_size
        self.volume_thresholds: Dict[str, float] = {}
        self.price_thresholds: Dict[str, float] = {}
        self.spread_thresholds: Dict[str, float] = {}
        self.impact_thresholds: Dict[str, float] = {}
            
    async def update_thresholds(self, symbol: str, df: pd.DataFrame, orderbook: Tuple[List, List]) -> None:
        """Update dynamic thresholds based on recent market conditions"""
        try:
            recent_data = df.tail(self.window_size)
            
            # Volume thresholds
            volume_std = recent_data['volume'].std()
            volume_mean = recent_data['volume'].mean()
            self.volume_thresholds[symbol] = {
                'spike': volume_mean + 2 * volume_std,
                'low': volume_mean - volume_std,
                'normal_range': (volume_mean - volume_std, volume_mean + volume_std)
            }
            
            # Price thresholds
            returns = recent_data['close'].pct_change()
            price_std = returns.std()
            self.price_thresholds[symbol] = {
                'volatility': price_std,
                'significant_move': price_std * 2,
                'extreme_move': price_std * 3
            }
            
            # Spread thresholds
            bids, asks = orderbook
            spreads = []
            for i in range(min(10, len(bids), len(asks))):
                bid_price = float(bids[i][0])
                ask_price = float(asks[i][0])
                if bid_price > 0:  # Prevent division by zero
                    spreads.append((ask_price - bid_price) / bid_price)
            
            if spreads:
                spread_mean = np.mean(spreads)
                spread_std = np.std(spreads) if len(spreads) > 1 else 0
                self.spread_thresholds[symbol] = {
                    'normal': spread_mean,
                    'wide': spread_mean + spread_std,
                    'extreme': spread_mean + 2 * spread_std
                }
            
            # Impact thresholds
            impact_samples = self.calculate_impact_samples(orderbook)
            if impact_samples:
                impact_mean = np.mean(impact_samples)
                impact_std = np.std(impact_samples) if len(impact_samples) > 1 else 0
                self.impact_thresholds[symbol] = {
                    'low': impact_mean - impact_std,
                    'medium': impact_mean,
                    'high': impact_mean + impact_std
                }
                
        except Exception as e:
            logger.error(f"Error updating dynamic thresholds: {str(e)}")
            self.reset_thresholds(symbol)

    def calculate_impact_samples(self, orderbook: Tuple[List, List]) -> List[float]:
        """Calculate market impact samples for different order sizes"""
        try:
            bids, asks = orderbook
            if not bids or not asks:
                return []
            
            base_size = float(np.mean([float(bid[1]) for bid in bids[:5]]))
            sample_sizes = [base_size * mult for mult in [0.5, 1, 2, 5]]
            
            impacts = []
            for size in sample_sizes:
                bid_impact = self.calculate_size_impact(bids, size)
                ask_impact = self.calculate_size_impact(asks, size)
                if bid_impact is not None and ask_impact is not None:
                    impacts.append((bid_impact + ask_impact) / 2)
            
            return impacts
                
        except Exception as e:
            logger.error(f"Error calculating impact samples: {str(e)}")
            return []

    @staticmethod
    def calculate_size_impact(orders: List, size: float) -> Optional[float]:
        """Calculate price impact for a given order size"""
        try:
            if not orders:
                return None
                
            cumulative_size = 0
            initial_price = float(orders[0][0])
            
            for price, order_size in orders:
                price = float(price)
                order_size = float(order_size)
                cumulative_size += order_size
                
                if cumulative_size >= size:
                    return abs(price - initial_price) / initial_price if initial_price > 0 else None
                    
            return None
                
        except Exception as e:
            logger.error(f"Error calculating size impact: {str(e)}")
            return None

    def reset_thresholds(self, symbol: str) -> None:
        """Reset thresholds to default values"""
        self.volume_thresholds[symbol] = {
            'spike': 2.0,
            'low': 0.5,
            'normal_range': (0.5, 2.0)
        }
        self.price_thresholds[symbol] = {
            'volatility': 0.02,
            'significant_move': 0.04,
            'extreme_move': 0.06
        }
        self.spread_thresholds[symbol] = {
            'normal': 0.001,
            'wide': 0.002,
            'extreme': 0.005
        }
        self.impact_thresholds[symbol] = {
            'low': 0.001,
            'medium': 0.002,
            'high': 0.005
        }

class EnhancedMarketMicrostructureBot:
    def __init__(self):
        # Core parameters - now using dynamic thresholds
        self.dynamic_thresholds = DynamicThresholds()
        self.short_period = 100
        self.long_period = 500
        self.timeframes = ["1m", "5m", "15m", "1h"]
        
        # Enhanced market microstructure parameters
        self.liquidity_levels = [0.001, 0.002, 0.005, 0.01, 0.02]
        self.institutional_threshold = 100000.0
        self.max_spread_threshold = 0.002
        self.min_liquidity_requirement = 1000000.0
        
        # Reversal detection parameters
        self.reversal_volume_threshold = 2.0
        self.reversal_price_threshold = 0.02
        self.reversal_confirmation_periods = 3
        self.reversal_momentum_threshold = 0.7
        self.ob_imbalance_threshold = 0.6  # New parameter for orderbook imbalance
        
        # Statistical validation parameters
        self.confidence_level = 0.95
        self.min_sample_size = 30
        self.correlation_threshold = 0.7
        
        # Error handling
        self.error_registry = defaultdict(int)
        self.max_errors = 5
        self.error_timeout = 300  # seconds

        # Performance optimization
        self.max_workers = 4
        self.retry_attempts = 3
        self.backoff_factor = 1.5
        self._setup_cache()

    def _setup_cache(self) -> None:
        """Setup caching parameters"""
        self.cache_timeouts = {
            'candles': 60,
            'orderbook': 5,
            'trades': 10,
            'analysis': 30
        }
        self.cache_data = {}

    @staticmethod
    def get_cache_key(prefix: str, **kwargs) -> str:
        """Generate consistent cache keys"""
        sorted_items = sorted(kwargs.items())
        return f"{prefix}:" + ":".join(f"{k}={v}" for k, v in sorted_items)

    async def get_cached_data(self, cache_key: str) -> Optional[Any]:
        """Get data from cache"""
        return self.cache_data.get(cache_key)

    async def set_cached_data(self, cache_key: str, data: Any, timeout: int) -> None:
        """Set data in cache"""
        self.cache_data[cache_key] = data

    async def handle_error(self, error_key: str, error: Exception) -> bool:
        """Handle errors with exponential backoff and circuit breaker"""
        try:
            self.error_registry[error_key] += 1
            error_count = self.error_registry[error_key]
            
            if error_count >= self.max_errors:
                logger.error(f"Circuit breaker triggered for {error_key}: {str(error)}")
                await self.notify_admin(f"Circuit breaker triggered for {error_key}")
                return False
                
            backoff_time = self.backoff_factor ** (error_count - 1)
            logger.warning(f"Error in {error_key}, attempt {error_count}: {str(error)}")
            await asyncio.sleep(backoff_time)
            
            return True
            
        except Exception as e:
            logger.error(f"Error in error handler: {str(e)}")
            return False

    async def notify_admin(self, message: str) -> None:
        """Notify admin of critical issues"""
        logger.critical(message)
        # Implement your notification logic here

    @staticmethod
    def validate_data(data: Any, data_type: str) -> bool:
        """Validate data integrity"""
        try:
            if data_type == 'dataframe':
                return isinstance(data, pd.DataFrame) and not data.empty
            elif data_type == 'orderbook':
                return (isinstance(data, tuple) and len(data) == 2 and
                       all(isinstance(x, list) for x in data))
            elif data_type == 'number':
                return isinstance(data, (int, float)) and not np.isnan(data)
            return False
        except Exception:
            return False

    async def analyze_market_microstructure(self, pair: str, orderbook: Tuple[List, List]) -> Dict[str, Any]:
        """Enhanced market microstructure analysis"""
        try:
            bids, asks = orderbook
            if not bids or not asks:
                return {}

            # Update dynamic thresholds based on current data
            # Fetch recent candles for dynamic threshold update
            timeframe = '1m'  # Using 1-minute data for thresholds
            df = await self.get_candles_df(pair, timeframe)
            await self.dynamic_thresholds.update_thresholds(pair, df, orderbook)

            analysis = {
                'liquidity_metrics': await self.analyze_liquidity_metrics(bids, asks),
                'order_flow_metrics': self.analyze_order_flow(bids, asks),
                'institutional_presence': await self.detect_institutional_presence(bids, asks),
                'market_impact': self.calculate_market_impact(bids, asks),
                'microstructure_signals': self.analyze_microstructure_signals(bids, asks),
                'orderbook_pressure': self.calculate_orderbook_pressure(bids, asks)
            }

            return analysis

        except Exception as e:
            logger.error(f"Error in market microstructure analysis: {str(e)}")
            return {}

    async def analyze_liquidity_metrics(self, bids: List, asks: List) -> Dict[str, Any]:
        """Advanced liquidity analysis"""
        liquidity_metrics = {
            'depth_profile': {},
            'resilience_metrics': {},
            'spread_analysis': {},
            'liquidity_score': 0.0
        }

        try:
            # Spread analysis
            best_bid = float(bids[0][0])
            best_ask = float(asks[0][0])
            spread = (best_ask - best_bid) / best_bid if best_bid > 0 else 0.0

            # Depth profile at different levels
            for level in self.liquidity_levels:
                depth_range = best_bid * level
                bid_depth = sum(float(bid[1]) for bid in bids if float(bid[0]) >= best_bid - depth_range)
                ask_depth = sum(float(ask[1]) for ask in asks if float(ask[0]) <= best_ask + depth_range)
                
                total_depth = bid_depth + ask_depth
                imbalance = (bid_depth - ask_depth) / total_depth if total_depth > 0 else 0

                liquidity_metrics['depth_profile'][f"level_{int(level*1000)}bps"] = {
                    'bid_depth': bid_depth,
                    'ask_depth': ask_depth,
                    'imbalance': imbalance
                }

            # Liquidity resilience
            liquidity_metrics['resilience_metrics'] = self.calculate_resilience_metrics(bids, asks)
            
            # Spread stability
            liquidity_metrics['spread_analysis'] = {
                'current_spread': spread,
                'spread_stability': self.calculate_spread_stability(bids, asks),
                'effective_spread': self.calculate_effective_spread(bids, asks)
            }

            # Overall liquidity score
            liquidity_metrics['liquidity_score'] = self.calculate_liquidity_score(liquidity_metrics)

            return liquidity_metrics

        except Exception as e:
            logger.error(f"Error in liquidity analysis: {str(e)}")
            return liquidity_metrics

    def calculate_resilience_metrics(self, bids: List, asks: List) -> Dict[str, Any]:
        """Calculate market resilience metrics"""
        try:
            best_bid = float(bids[0][0])
            best_ask = float(asks[0][0])
            mid_price = (best_bid + best_ask) / 2

            # Calculate order book slope
            bid_slope = self.calculate_order_book_slope(bids, mid_price, side='bid')
            ask_slope = self.calculate_order_book_slope(asks, mid_price, side='ask')

            # Calculate price impact for standard volume
            standard_volume = 100000  # Adjust based on your needs
            bid_impact = self.calculate_price_impact(bids, standard_volume, side='bid')
            ask_impact = self.calculate_price_impact(asks, standard_volume, side='ask')

            return {
                'bid_slope': bid_slope,
                'ask_slope': ask_slope,
                'bid_impact': bid_impact,
                'ask_impact': ask_impact,
                'symmetry': abs(bid_slope - ask_slope) / max(bid_slope, ask_slope) if max(bid_slope, ask_slope) > 0 else 0.0,
                'recovery_factor': self.estimate_recovery_factor(bids, asks)
            }

        except Exception as e:
            logger.error(f"Error calculating resilience metrics: {str(e)}")
            return {}

    def calculate_order_book_slope(self, orders: List, mid_price: float, side: str) -> float:
        """Calculate order book slope"""
        try:
            prices = []
            volumes = []
            cumulative_volume = 0

            for price, volume in orders[:10]:  # Use top 10 levels
                price = float(price)
                volume = float(volume)
                prices.append(abs(price - mid_price) / mid_price)  # Normalize prices
                cumulative_volume += volume
                volumes.append(cumulative_volume)

            if not prices or not volumes:
                return 0.0

            # Calculate slope using numpy's polyfit
            slope, _ = np.polyfit(volumes, prices, deg=1)
            return abs(slope)

        except Exception as e:
            logger.error(f"Error calculating order book slope: {str(e)}")
            return 0.0

    def calculate_price_impact(self, orders: List, volume: float, side: str) -> float:
        """Calculate price impact for given volume"""
        try:
            cumulative_volume = 0
            initial_price = float(orders[0][0])

            for price, size in orders:
                price = float(price)
                size = float(size)
                cumulative_volume += size

                if cumulative_volume >= volume:
                    # Calculate weighted average price impact
                    return abs(price - initial_price) / initial_price if initial_price > 0 else 0.0

            return 1.0  # Maximum impact if order size exceeds available liquidity

        except Exception as e:
            logger.error(f"Error calculating price impact: {str(e)}")
            return 1.0

    def estimate_recovery_factor(self, bids: List, asks: List) -> float:
        """Estimate market recovery factor"""
        try:
            best_bid = float(bids[0][0])
            best_ask = float(asks[0][0])
            mid_price = (best_bid + best_ask) / 2

            # Calculate total available liquidity within 0.5% of mid price
            near_liquidity = sum(float(bid[1]) for bid in bids if float(bid[0]) >= mid_price * 0.995) + \
                           sum(float(ask[1]) for ask in asks if float(ask[0]) <= mid_price * 1.005)

            # Calculate total available liquidity within 2% of mid price
            far_liquidity = sum(float(bid[1]) for bid in bids if float(bid[0]) >= mid_price * 0.98) + \
                          sum(float(ask[1]) for ask in asks if float(ask[0]) <= mid_price * 1.02)

            if far_liquidity == 0:
                return 0.0

            return near_liquidity / far_liquidity

        except Exception as e:
            logger.error(f"Error estimating recovery factor: {str(e)}")
            return 0.0

    def calculate_liquidity_score(self, metrics: Dict[str, Any]) -> float:
        """Calculate overall liquidity score"""
        try:
            depth_scores = [
                profile['bid_depth'] + profile['ask_depth']
                for profile in metrics['depth_profile'].values()
            ]
            depth_score = np.mean(depth_scores) if depth_scores else 0.0

            spread = metrics['spread_analysis']['current_spread']
            liquidity_score = depth_score / (spread + 1e-8)  # Prevent division by zero
            return liquidity_score

        except Exception as e:
            logger.error(f"Error calculating liquidity score: {str(e)}")
            return 0.0

    def calculate_spread_stability(self, bids: List, asks: List) -> float:
        """Calculate spread stability metric"""
        try:
            spreads = []
            for i in range(min(10, len(bids), len(asks))):
                bid_price = float(bids[i][0])
                ask_price = float(asks[i][0])
                spreads.append((ask_price - bid_price) / bid_price if bid_price > 0 else 0)
                
            return float(np.std(spreads)) if spreads else 1.0

        except Exception as e:
            logger.error(f"Error calculating spread stability: {str(e)}")
            return 1.0

    def calculate_effective_spread(self, bids: List, asks: List) -> float:
        """Calculate effective spread considering market impact"""
        try:
            if not bids or not asks:
                return 0.0
                
            top_bid = float(bids[0][0])
            top_ask = float(asks[0][0])
            mid_price = (top_bid + top_ask) / 2 if (top_bid + top_ask) > 0 else 0.0

            return float((top_ask - top_bid) / mid_price) if mid_price > 0 else 0.0

        except Exception as e:
            logger.error(f"Error calculating effective spread: {str(e)}")
            return 0.0

    def analyze_order_flow(self, symbol: str) -> Dict[str, Any]:
        """Analyze order flow dynamics"""
        try:
        # Query the most recent snapshot for the given symbol
            current_snapshot = (
                OrderBook.objects
                .filter(symbol=symbol)
                .latest('timestamp')
            )

        # Fetch the snapshot before the current one
            previous_snapshot = (
                OrderBook.objects
                .filter(symbol=symbol, timestamp__lt=current_snapshot.timestamp)
                .order_by('-timestamp')
                .first()
            )

        # If there's no previous snapshot, we can't compute changes
            if not previous_snapshot:
                logger.warning("Not enough historical data for analysis")
                return {
                    'aggressive_buying': False,
                    'aggressive_selling': False,
                    'bid_size_change': 0.0,
                    'ask_size_change': 0.0
                }

        # Calculate bid and ask size changes
            current_bid_size = sum(float(bid['amount']) for bid in current_snapshot.bids)
            current_ask_size = sum(float(ask['amount']) for ask in current_snapshot.asks)
            previous_bid_size = sum(float(bid['amount']) for bid in previous_snapshot.bids)
            previous_ask_size = sum(float(ask['amount']) for ask in previous_snapshot.asks)

            bid_size_change = current_bid_size - previous_bid_size
            ask_size_change = current_ask_size - previous_ask_size

        # Determine if there's aggressive buying or selling taking place
            aggressive_buying = bid_size_change > 0 and ask_size_change < 0
            aggressive_selling = bid_size_change < 0 and ask_size_change > 0

            order_flow_metrics = {
                'aggressive_buying': aggressive_buying,
                'aggressive_selling': aggressive_selling,
                'bid_size_change': bid_size_change,
                'ask_size_change': ask_size_change
            }

            return order_flow_metrics

        except Exception as e:
            logger.error(f"Error analyzing order flow: {str(e)}")
            return {}

    async def detect_institutional_presence(self, bids: List, asks: List) -> Dict[str, Any]:
        """Detect and analyze institutional order flow"""
        try:
            institutional_presence = {
                'large_orders': {'bids': [], 'asks': []},
                'clustering': {'bids': {}, 'asks': {}},
                'iceberg_probability': 0.0,
                'smart_routing_suggestion': None
            }

            # Detect large orders
            for side, orders in [('bids', bids), ('asks', asks)]:
                large_orders = []
                price_clusters = defaultdict(float)

                for price, size in orders:
                    price, size = float(price), float(size)
                    if size >= self.institutional_threshold:
                        large_orders.append({'price': price, 'size': size})
                    
                    # Round price to analyze clustering
                    price_bucket = round(price, 3)
                    price_clusters[price_bucket] += size

                institutional_presence['large_orders'][side] = large_orders
                
                # Analyze clustering
                significant_clusters = {
                    price: volume for price, volume in price_clusters.items()
                    if volume >= self.institutional_threshold
                }
                institutional_presence['clustering'][side] = significant_clusters

            # Detect potential iceberg orders
            institutional_presence['iceberg_probability'] = self.detect_iceberg_orders(bids, asks)

            # Generate smart routing suggestion
            institutional_presence['smart_routing_suggestion'] = self.generate_smart_routing_suggestion(institutional_presence)

            return institutional_presence

        except Exception as e:
            logger.error(f"Error detecting institutional presence: {str(e)}")
            return {}

    def detect_iceberg_orders(self, bids: List, asks: List) -> float:
        """Detect potential iceberg orders"""
        try:
            # Analyze order size patterns
            bid_sizes = [float(bid[1]) for bid in bids[:5]]  # Top 5 levels
            ask_sizes = [float(ask[1]) for ask in asks[:5]]

            # Calculate size variation
            bid_variation = np.std(bid_sizes) / np.mean(bid_sizes) if np.mean(bid_sizes) > 0 else 0
            ask_variation = np.std(ask_sizes) / np.mean(ask_sizes) if np.mean(ask_sizes) > 0 else 0

            # High variation might indicate iceberg orders
            iceberg_probability = max(bid_variation, ask_variation) / 2

            return min(1.0, iceberg_probability)

        except Exception as e:
            logger.error(f"Error detecting iceberg orders: {str(e)}")
            return 0.0

    def generate_smart_routing_suggestion(self, institutional_analysis: Dict[str, Any]) -> Dict[str, Any]:
        """Generate smart order routing suggestions"""
        try:
            suggestion = {
                'preferred_venue': 'primary',
                'order_type': 'limit',
                'execution_style': 'normal',
                'urgency': 'medium'
            }

            # Analyze institutional presence
            bid_clusters = len(institutional_analysis['clustering']['bids'])
            ask_clusters = len(institutional_analysis['clustering']['asks'])
            large_orders = len(institutional_analysis['large_orders']['bids']) + \
                         len(institutional_analysis['large_orders']['asks'])

            # Adjust suggestion based on market conditions
            if large_orders > 5:
                suggestion['execution_style'] = 'careful'
                suggestion['urgency'] = 'low'
                suggestion['order_type'] = 'iceberg'

            if bid_clusters > 3 or ask_clusters > 3:
                suggestion['execution_style'] = 'aggressive'
                suggestion['urgency'] = 'high'

            if institutional_analysis['iceberg_probability'] > 0.7:
                suggestion['order_type'] = 'peg'
                suggestion['execution_style'] = 'smart'

            return suggestion

        except Exception as e:
            logger.error(f"Error generating smart routing suggestion: {str(e)}")
            return {}

    async def calculate_market_impact(self, bids: List, asks: List) -> Dict[str, Any]:
        """Calculate market impact metrics with detailed analysis"""
        try:
            # Simulate execution of different order sizes
            order_sizes = [10000, 50000, 100000, 500000]  # Example sizes in base currency
            price_impacts = []
            for size in order_sizes:
                buy_impact = self.simulate_order_execution(asks, size, 'buy')
                sell_impact = self.simulate_order_execution(bids, size, 'sell')
                avg_impact = (buy_impact + sell_impact) / 2
                price_impacts.append({'size': size, 'impact': avg_impact})

            # Fit a linear regression to estimate impact per unit size
            sizes = np.array([x['size'] for x in price_impacts]).reshape(-1, 1)
            impacts = np.array([x['impact'] for x in price_impacts])

            from sklearn.linear_model import LinearRegression

            model = LinearRegression()
            model.fit(sizes, impacts)
            impact_per_unit = model.coef_[0]
            intercept = model.intercept_

            # Statistical significance
            from scipy import stats

            slope, intercept, r_value, p_value, std_err = stats.linregress(sizes.flatten(), impacts)
            significance = 1 - p_value  # Closer to 1 means more significant

            market_impact = {
                'impact_per_unit': float(impact_per_unit),
                'intercept': float(intercept),
                'r_squared': r_value ** 2,
                'significance': significance,
                'estimated_impact': price_impacts
            }

            return market_impact

        except Exception as e:
            logger.error(f"Error calculating market impact: {str(e)}")
            return {}

    def simulate_order_execution(self, orders: List, size: float, side: str) -> float:
        """Simulate order execution to estimate price impact"""
        try:
            cumulative_size = 0.0
            initial_price = float(orders[0][0])
            for price, order_size in orders:
                price = float(price)
                order_size = float(order_size)
                cumulative_size += order_size
                if cumulative_size >= size:
                    impact = abs(price - initial_price) / initial_price
                    return impact
            # If order size exceeds available liquidity
            return 1.0  # Maximum impact
        except Exception as e:
            logger.error(f"Error simulating order execution: {str(e)}")
            return 1.0

    def analyze_microstructure_signals(self, bids: List, asks: List) -> Dict[str, Any]:
        """Analyze microstructure signals for patterns indicating trading opportunities"""
        try:
            microstructure_signals = {}

        # Detect Bid-Ask Bounce
            bid_prices = [float(bid[0]) for bid in bids[:5]]
            ask_prices = [float(ask[0]) for ask in asks[:5]]
            spread = ask_prices[0] - bid_prices[0]

            if spread < self.dynamic_thresholds.spread_thresholds.get('normal', 0.001):
                microstructure_signals['bid_ask_bounce'] = True

        # Detect Book Pressure
            bid_sizes = [float(bid[1]) for bid in bids[:5]]
            ask_sizes = [float(ask[1]) for ask in asks[:5]]
            bid_volume = sum(bid_sizes)
            ask_volume = sum(ask_sizes)
            pressure = (bid_volume - ask_volume) / (bid_volume + ask_volume)

            if pressure > 0.5:
                microstructure_signals['buy_pressure'] = True
            elif pressure < -0.5:
                microstructure_signals['sell_pressure'] = True

        # Detect Quote Stuffing
            if self.detect_quote_stuffing(bids, asks):
                microstructure_signals['quote_stuffing_detected'] = True

        # Detect Layers (Spoofing)
            if self.detect_layers(bids, asks):
                microstructure_signals['layers_detected'] = True

            return microstructure_signals

        except Exception as e:
            logger.error(f"Error analyzing microstructure signals: {str(e)}")
            return {}
    
    def detect_quote_stuffing(self, symbol: str, lookback_period: int = 1, order_threshold: int = 100) -> bool:
        """
        Detect quote stuffing by rapid order book updates.
    
        :param symbol: The trading symbol to analyze.
        :param lookback_period: Number of minutes to look back for analyzing frequent updates.
        :param order_threshold: The number of order changes considered indicative of quote stuffing.
        :return: True if quote stuffing is detected, otherwise False.
        """
        try:
            # Calculate the timestamp limit for the lookback period
            now = timezone.now()
            lookback_time = now - timedelta(minutes=lookback_period)

        # Utilize cached data when possible
            cache_key = f"{symbol}_{lookback_time.isoformat()}"
            if cache_key in recent_snapshots_cache:
                recent_snapshots = recent_snapshots_cache[cache_key]
            else:
            # Query recent order book snapshots within the lookback period
                recent_snapshots = list(OrderBook.objects.filter(
                    symbol=symbol,
                    timestamp__gte=lookback_time
                ).order_by('timestamp'))
                recent_snapshots_cache[cache_key] = recent_snapshots  # Update cache

        # If there are too few snapshots, quote stuffing is not detectable
            if len(recent_snapshots) < 2:
                return False

        # Initialize order update count
            order_update_count = 0

        # Analyze snapshots for characteristics of quote stuffing
            for i in range(1, len(recent_snapshots)):
                previous_snapshot = recent_snapshots[i - 1]
                current_snapshot = recent_snapshots[i]

            # Calculate changes in price levels and sizes
                bid_changes = set(map(tuple, current_snapshot.bids)) ^ set(map(tuple, previous_snapshot.bids))
                ask_changes = set(map(tuple, current_snapshot.asks)) ^ set(map(tuple, previous_snapshot.asks))

            # Detect rapid significant changes
                order_update_count += len(bid_changes) + len(ask_changes)

            # Immediately return True if the threshold is surpassed
                if order_update_count > order_threshold:
                    return True

            return False

        except Exception as e:
            logger.error(f"Error detecting quote stuffing: {str(e)}")
            return False
    
    def detect_layers(self, bids: List, asks: List) -> bool:
        """Detect layering or spoofing activities."""
        try:
            # Check for unusually large sizes at different price levels
            bid_sizes = [float(bid[1]) for bid in bids]
            ask_sizes = [float(ask[1]) for ask in asks]

            threshold = np.mean(bid_sizes + ask_sizes) + 2 * np.std(bid_sizes + ask_sizes)

            for i in range(1, len(bids)):
                if float(bids[i][1]) > threshold:
                    return True
            for i in range(1, len(asks)):
                if float(asks[i][1]) > threshold:
                    return True
            return False
        except Exception as e:
            logger.error(f"Error detecting layers: {str(e)}")
            return False

    def calculate_orderbook_pressure(self, bids: List, asks: List) -> Dict[str, Any]:
        """Calculate orderbook pressure"""
        try:
            bid_volume = sum(float(bid[1]) for bid in bids[:10])
            ask_volume = sum(float(ask[1]) for ask in asks[:10])
            total_volume = bid_volume + ask_volume

            pressure = (bid_volume - ask_volume) / total_volume if total_volume > 0 else 0.0

            return {
                'pressure': pressure,
                'side': 'buy' if pressure > 0 else 'sell' if pressure < 0 else 'neutral'
            }

        except Exception as e:
            logger.error(f"Error calculating orderbook pressure: {str(e)}")
            return {}

    async def analyze_institutional_activity(self, orderbook: Tuple[List, List], volume_analysis: Dict[str, Any]) -> Dict[str, Any]:
        """Analyze institutional trading activity"""
        try:
            bids, asks = orderbook
            institutional_analysis = await self.detect_institutional_presence(bids, asks)
            # Additional analysis can be added here
            return institutional_analysis

        except Exception as e:
            logger.error(f"Error analyzing institutional activity: {str(e)}")
            return {}

    def calculate_institutional_score(self, institutional_analysis: Dict[str, Any]) -> float:
        """Calculate institutional score"""
        try:
            # Placeholder implementation
            large_orders_count = len(institutional_analysis['large_orders']['bids']) + \
                                 len(institutional_analysis['large_orders']['asks'])
            iceberg_probability = institutional_analysis.get('iceberg_probability', 0.0)
            institutional_score = (large_orders_count * 0.1) + (iceberg_probability * 0.9)
            return min(institutional_score, 1.0)

        except Exception as e:
            logger.error(f"Error calculating institutional score: {str(e)}")
            return 0.0

    async def generate_signal(self, pair: str, timeframe: str) -> Tuple[SignalType, Optional[SignalMetrics]]:
        """Generate comprehensive trading signal"""
        try:
            # Fetch required data
            df = await self.get_candles_df(pair, timeframe)
            if df.empty:
                return SignalType.NO_SIGNAL, None

            orderbook = await self.get_orderbook(pair)
            if not orderbook[0] or not orderbook[1]:
                return SignalType.NO_SIGNAL, None

            # Perform comprehensive analysis
            current_price = float(df['close'].iloc[-1])

            # Adjust calculation for price_24h_ago
            now = df.index[-1]
            price_24h_ago_time = now - timedelta(hours=24)
            df_24h_ago = df[df.index <= price_24h_ago_time]
            if not df_24h_ago.empty:
                price_24h_ago = float(df_24h_ago['close'].iloc[-1])
            else:
                price_24h_ago = float(df['close'].iloc[0])
            price_24h_change = ((current_price - price_24h_ago) / price_24h_ago)

            # Volume analysis
            volume_analysis = await self.analyze_volume_metrics(df)
            # Orderbook analysis
            orderbook_analysis = await self.analyze_market_microstructure(pair, orderbook)
            # Market regime
            market_regime = await self.detect_market_regime(df, orderbook_analysis)
            # Market impact
            market_impact = await self.calculate_detailed_market_impact(orderbook)
            # Reversal analysis
            reversal_analysis = await self.detect_trend_reversal(df, orderbook, volume_analysis)
            # Institutional analysis
            institutional_analysis = await self.analyze_institutional_activity(orderbook, volume_analysis)

            # Generate signal type
            signal_type, contributing_factors = await self.determine_signal_type(
                df, orderbook_analysis, volume_analysis, market_regime,
                reversal_analysis, institutional_analysis
            )
            if signal_type == SignalType.NO_SIGNAL:
                return signal_type, None

            # Calculate risk levels
            stop_loss, take_profit = await self.calculate_risk_levels(
                signal_type, current_price, orderbook_analysis, market_impact
            )

            # Calculate success rate
            success_rate = await self.calculate_success_probability(
                signal_type, volume_analysis, orderbook_analysis,
                market_regime, institutional_analysis
            )

            # Prepare signal metrics
            metrics = SignalMetrics(
                signal_time=timezone.now(),
                chart_timestamp=df.index[-1],
                symbol=pair,
                current_price=current_price,
                price_24h_change=price_24h_change * 100,
                signal_type=signal_type,
                stop_loss=stop_loss,
                take_profit=take_profit,
                success_rate=success_rate * 100,  # Convert to percentage
                result="Pending",
                volume_analysis=volume_analysis,
                orderbook_analysis=orderbook_analysis,
                technical_analysis={},  # For now, empty as we are focusing on orderbook
                market_regime=market_regime,
                market_impact=market_impact,
                reversal_analysis=reversal_analysis,
                institutional_analysis=institutional_analysis,
                contributing_factors=contributing_factors
            )

            # Save signal to database
            await self.save_signal(metrics)

            return signal_type, metrics
        except Exception as e:
            logger.error(f"Error in generate_signal: {str(e)}")
            return SignalType.NO_SIGNAL, None

    async def analyze_volume_metrics(self, df: pd.DataFrame) -> Dict[str, Any]:
        """Analyze volume patterns and characteristics"""
        try:
            recent_volume = df['volume'].tail(self.short_period)
            volume_sma = recent_volume.mean()
            volume_std = recent_volume.std()
            volume_zscore = (recent_volume.iloc[-1] - volume_sma) / volume_std if volume_std > 0 else 0.0

            return {
                'current_volume': float(recent_volume.iloc[-1]),
                'volume_sma': float(volume_sma),
                'volume_std': float(volume_std),
                'volume_zscore': float(volume_zscore),
                'volume_trend': self.calculate_volume_trend(recent_volume),
                'volume_concentration': self.analyze_volume_concentration(df),
                'buying_pressure': self.calculate_buying_pressure(df),
                'volume_profile': self.analyze_volume_profile(df)
            }
        except Exception as e:
            logger.error(f"Error in volume analysis: {str(e)}")
            return {}

    def calculate_volume_trend(self, volumes: pd.Series) -> str:
        """Analyze volume trend"""
        try:
            volume_sma = volumes.rolling(window=20).mean()
            current_vol = volumes.iloc[-1]
            avg_vol = volume_sma.iloc[-1] if volume_sma.iloc[-1] > 0 else volumes.mean()

            if current_vol > avg_vol * 2:
                return "strong_increasing"
            elif current_vol > avg_vol * 1.5:
                return "increasing"
            elif current_vol < avg_vol * 0.5:
                return "strong_decreasing"
            elif current_vol < avg_vol * 0.75:
                return "decreasing"
            else:
                return "stable"

        except Exception as e:
            logger.error(f"Error calculating volume trend: {str(e)}")
            return "unknown"

    def analyze_volume_concentration(self, df: pd.DataFrame) -> Dict[str, Any]:
        """Analyze volume concentration patterns"""
        try:
            recent_df = df.tail(100)
            price_range = recent_df['close'].max() - recent_df['close'].min()
            bins = np.linspace(recent_df['close'].min(), recent_df['close'].max(), num=10)
            volume_price_levels = pd.cut(recent_df['close'], bins=bins, include_lowest=True)
            volume_concentration = recent_df.groupby(volume_price_levels)['volume'].sum()

            return {
                'high_volume_levels': volume_concentration.nlargest(3).index.astype(str).tolist(),
                'concentration_ratio': float(volume_concentration.max() / volume_concentration.sum()) if volume_concentration.sum() > 0 else 0.0,
                'distribution': volume_concentration.to_dict()
            }

        except Exception as e:
            logger.error(f"Error analyzing volume concentration: {str(e)}")
            return {}

    def calculate_buying_pressure(self, df: pd.DataFrame) -> float:
        """Calculate buying pressure based on volume and price movement"""
        try:
            recent_df = df.tail(20)
            up_volume = recent_df[recent_df['close'] > recent_df['open']]['volume'].sum()
            down_volume = recent_df[recent_df['close'] < recent_df['open']]['volume'].sum()
            total_volume = up_volume + down_volume

            if total_volume == 0:
                return 0.5

            return float(up_volume / total_volume)

        except Exception as e:
            logger.error(f"Error calculating buying pressure: {str(e)}")
            return 0.5

    def analyze_volume_profile(self, df: pd.DataFrame) -> Dict[str, Any]:
        """Analyze volume profile characteristics"""
        try:
            recent_df = df.tail(100)

            profile = {
                'avg_volume': float(recent_df['volume'].mean()),
                'vol_std': float(recent_df['volume'].std()),
                'volume_pods': {},
                'volume_trend_strength': 0.0
            }

            # Calculate volume Points of Distribution
            price_levels = pd.qcut(recent_df['close'], q=10, duplicates='drop')
            volume_pods = recent_df.groupby(price_levels)['volume'].agg(['sum', 'count'])
            profile['volume_pods'] = volume_pods.to_dict()

            # Calculate volume trend strength
            volume_sma = recent_df['volume'].rolling(window=20).mean()
            if not volume_sma.empty and volume_sma.iloc[-1] > 0:
                volume_trend = recent_df['volume'].iloc[-1] / volume_sma.iloc[-1]
            else:
                volume_trend = 1.0
            profile['volume_trend_strength'] = float(volume_trend)

            return profile

        except Exception as e:
            logger.error(f"Error analyzing volume profile: {str(e)}")
            return {}

    async def detect_market_regime(self, df: pd.DataFrame, orderbook_analysis: Dict[str, Any]) -> MarketRegime:
        """Detect current market regime"""
        try:
            # Calculate volatility
            returns = df['close'].pct_change()
            volatility = returns.std() * np.sqrt(len(df))

            # Calculate trend strength
            price_sma = df['close'].rolling(window=20).mean()
            if price_sma.iloc[-1] > 0:
                trend_strength = abs(df['close'].iloc[-1] - price_sma.iloc[-1]) / price_sma.iloc[-1]
            else:
                trend_strength = 0.0

            # Get liquidity score
            liquidity_score = orderbook_analysis.get('liquidity_metrics', {}).get('liquidity_score', 0.0)

            # Determine regime type
            if volatility > 0.02 and liquidity_score < 0.5:
                regime_type = "high_volatility_low_liquidity"
                confidence = 0.8
            elif volatility > 0.02:
                regime_type = "high_volatility"
                confidence = 0.85
            elif liquidity_score < 0.5:
                regime_type = "low_liquidity"
                confidence = 0.75
            elif trend_strength > 0.02:
                regime_type = "trending"
                confidence = 0.9
            else:
                regime_type = "normal"
                confidence = 0.7

            market_regime = MarketRegime(
                volatility=float(volatility),
                trend_strength=float(trend_strength),
                liquidity_score=float(liquidity_score),
                regime_type=regime_type,
                confidence=confidence,
                volume_profile={},
                market_efficiency=0.0,
                regime_change_probability=0.0
            )

            return market_regime

        except Exception as e:
            logger.error(f"Error detecting market regime: {str(e)}")
            return MarketRegime(0, 0, 0, "unknown", 0, {}, 0, 0)

    async def calculate_detailed_market_impact(self, orderbook: Tuple[List, List]) -> MarketImpact:
        """Calculate detailed market impact metrics"""
        try:
            bids, asks = orderbook
            if not bids or not asks:
                return MarketImpact(0, 0, 0, 0, 0, 0, 0)

            # Calculate immediate price impact for standard size
            standard_size = 10000  # Adjust based on your needs
            bid_impact = self.calculate_price_impact(bids, standard_size, 'bid')
            ask_impact = self.calculate_price_impact(asks, standard_size, 'ask')
            price_impact = (bid_impact + ask_impact) / 2

            # Calculate volume impact
            bid_volume = sum(float(bid[1]) for bid in bids[:10])
            ask_volume = sum(float(ask[1]) for ask in asks[:10])
            total_volume = bid_volume + ask_volume
            volume_impact = standard_size / total_volume if total_volume > 0 else 1.0

            # Estimate slippage
            slippage_estimate = price_impact * (1 + volume_impact)

            # Estimate recovery time based on volume and impact
            recovery_time = self.estimate_recovery_time(slippage_estimate, volume_impact)

            market_impact = MarketImpact(
                price_impact=float(price_impact),
                volume_impact=float(volume_impact),
                slippage_estimate=float(slippage_estimate),
                recovery_time=float(recovery_time),
                impact_asymmetry=0.0,
                market_resilience=0.0,
                depth_score=0.0
            )

            return market_impact

        except Exception as e:
            logger.error(f"Error calculating market impact: {str(e)}")
            return MarketImpact(0, 0, 0, 0, 0, 0, 0)

    def estimate_recovery_time(self, slippage: float, volume_impact: float) -> float:
        """Estimate market recovery time in minutes"""
        try:
            base_recovery = 5  # Base recovery time in minutes
            impact_multiplier = 1 + (slippage * 10)  # Higher slippage = longer recovery
            volume_multiplier = 1 + (volume_impact * 5)  # Higher volume impact = longer recovery

            return base_recovery * impact_multiplier * volume_multiplier

        except Exception as e:
            logger.error(f"Error estimating recovery time: {str(e)}")
            return 5.0

    async def detect_trend_reversal(
        self, df: pd.DataFrame, orderbook: Tuple[List, List],
        volume_analysis: Dict[str, Any]
    ) -> Dict[str, Any]:
        """Detect potential trend reversals"""
        try:
            bids, asks = orderbook
            current_price = float(df['close'].iloc[-1])

            analysis = {
                'reversal_detected': False,
                'direction': None,
                'confidence': 0,
                'supporting_factors': []
            }

            # Volume surge check
            volume_zscore = volume_analysis.get('volume_zscore', 0.0)
            if abs(volume_zscore) > self.reversal_volume_threshold:
                analysis['supporting_factors'].append('volume_surge')

            # Orderbook imbalance check
            bid_volume = sum(float(bid[1]) for bid in bids[:10])
            ask_volume = sum(float(ask[1]) for ask in asks[:10])
            ob_imbalance = (bid_volume - ask_volume) / (bid_volume + ask_volume) if (bid_volume + ask_volume) > 0 else 0.0

            if abs(ob_imbalance) > self.ob_imbalance_threshold:
                analysis['supporting_factors'].append('orderbook_imbalance')

            # Price movement check
            recent_returns = df['close'].pct_change().tail(self.reversal_confirmation_periods)
            price_change = recent_returns.sum()

            if abs(price_change) > self.reversal_price_threshold:
                analysis['supporting_factors'].append('price_movement')

            # Determine reversal
            if len(analysis['supporting_factors']) >= 2:
                analysis['reversal_detected'] = True
                analysis['direction'] = 'bullish' if ob_imbalance > 0 else 'bearish'
                analysis['confidence'] = min(0.9, 0.6 + len(analysis['supporting_factors']) * 0.1)

            return analysis

        except Exception as e:
            logger.error(f"Error detecting trend reversal: {str(e)}")
            return {'reversal_detected': False}

    async def determine_signal_type(
        self, df: pd.DataFrame, orderbook_analysis: Dict[str, Any],
        volume_analysis: Dict[str, Any], market_regime: MarketRegime,
        reversal_analysis: Dict[str, Any], institutional_analysis: Dict[str, Any]
    ) -> Tuple[SignalType, List[str]]:
        """Determine signal type based on comprehensive analysis"""
        try:
            contributing_factors = []

            # Calculate component scores
            volume_score = self.calculate_volume_score(volume_analysis)
            orderbook_score = self.calculate_orderbook_score(orderbook_analysis)
            institutional_score = self.calculate_institutional_score(institutional_analysis)

            # Combined weighted score
            total_score = (
                volume_score * 0.4 +
                orderbook_score * 0.4 +
                institutional_score * 0.2
            )

            # Check for reversals first
            if reversal_analysis.get('reversal_detected'):
                if reversal_analysis['direction'] == 'bullish':
                    contributing_factors.append('reversal_bullish')
                    return SignalType.REVERSAL_BULLISH, contributing_factors
                contributing_factors.append('reversal_bearish')
                return SignalType.REVERSAL_BEARISH, contributing_factors

            # Check for range-bound market
            if self.is_range_bound(df, market_regime):
                contributing_factors.append('range_bound')
                return SignalType.NEUTRAL, contributing_factors

            # Determine signal based on total score
            if total_score > 0.8:
                contributing_factors.append('strong_buy')
                return SignalType.STRONG_BUY, contributing_factors
            elif total_score > 0.6:
                contributing_factors.append('buy')
                return SignalType.BUY, contributing_factors
            elif total_score < -0.8:
                contributing_factors.append('strong_sell')
                return SignalType.STRONG_SELL, contributing_factors
            elif total_score < -0.6:
                contributing_factors.append('sell')
                return SignalType.SELL, contributing_factors

            return SignalType.NO_SIGNAL, contributing_factors

        except Exception as e:
            logger.error(f"Error determining signal type: {str(e)}")
            return SignalType.NO_SIGNAL, []

    def calculate_volume_score(self, volume_analysis: Dict[str, Any]) -> float:
        """Calculate volume score"""
        try:
            volume_zscore = volume_analysis.get('volume_zscore', 0.0)
            volume_trend = volume_analysis.get('volume_trend', 'stable')
            buying_pressure = volume_analysis.get('buying_pressure', 0.5)

            if volume_trend in ['strong_increasing', 'increasing']:
                trend_score = 1.0
            elif volume_trend in ['strong_decreasing', 'decreasing']:
                trend_score = -1.0
            else:
                trend_score = 0.0

            volume_score = volume_zscore / 3 + trend_score * 0.5 + (buying_pressure - 0.5) * 2

            return max(min(volume_score, 1.0), -1.0)

        except Exception as e:
            logger.error(f"Error calculating volume score: {str(e)}")
            return 0.0

    def calculate_orderbook_score(self, orderbook_analysis: Dict[str, Any]) -> float:
        """Calculate orderbook score"""
        try:
            ob_pressure = orderbook_analysis.get('orderbook_pressure', {}).get('pressure', 0.0)
            liquidity_score = orderbook_analysis.get('liquidity_metrics', {}).get('liquidity_score', 0.0)
            microstructure_signals = orderbook_analysis.get('microstructure_signals', {})

            # Calculate contribution from microstructure signals
            microstructure_score = 0.0

        # Adjust the microstructure_score based on detected signals
            if microstructure_signals.get('bid_ask_bounce'):
                microstructure_score += 0.1  # Assign an appropriate value

            if microstructure_signals.get('buy_pressure'):
                microstructure_score += 0.1

            if microstructure_signals.get('sell_pressure'):
                microstructure_score -= 0.1

            if microstructure_signals.get('quote_stuffing_detected'):
                microstructure_score -= 0.2

            if microstructure_signals.get('layers_detected'):
                microstructure_score -= 0.2

        # Normalize microstructure_score to a reasonable range if needed
            microstructure_score = max(min(microstructure_score, 0.5), -0.5)

        # Calculate the final orderbook score with the contributions
            orderbook_score = ob_pressure * 0.5 + liquidity_score * 0.3 + microstructure_score * 0.2

        # Ensure the score is between -1.0 and 1.0
            return max(min(orderbook_score, 1.0), -1.0)


        except Exception as e:
            logger.error(f"Error calculating orderbook score: {str(e)}")
            return 0.0

    async def calculate_success_probability(
        self, signal_type: SignalType, volume_analysis: Dict[str, Any],
        orderbook_analysis: Dict[str, Any], market_regime: MarketRegime,
        institutional_analysis: Dict[str, Any]
    ) -> float:
        """Calculate probability of signal success"""
        try:
            # Base probability from volume analysis
            volume_probability = min(1.0, volume_analysis.get('volume_zscore', 0.0) / 3)

            # Orderbook confidence
            ob_confidence = orderbook_analysis.get('liquidity_metrics', {}).get('liquidity_score', 0.0)

            # Institutional confidence
            inst_confidence = institutional_analysis.get('iceberg_probability', 0.0)

            # Market regime adjustment
            regime_multiplier = 1.0
            if market_regime.regime_type == "high_volatility":
                regime_multiplier = 0.8
            elif market_regime.regime_type == "low_liquidity":
                regime_multiplier = 0.7

            # Combined probability
            base_probability = (
                volume_probability * 0.4 +
                ob_confidence * 0.4 +
                inst_confidence * 0.2
            ) * regime_multiplier

            # Adjust based on signal type
            if signal_type in [SignalType.STRONG_BUY, SignalType.STRONG_SELL]:
                base_probability *= 1.2
            elif signal_type in [SignalType.REVERSAL_BULLISH, SignalType.REVERSAL_BEARISH]:
                base_probability *= 0.9

            return min(0.95, max(0.05, base_probability))

        except Exception as e:
            logger.error(f"Error calculating success probability: {str(e)}")
            return 0.5

    async def calculate_risk_levels(
        self, signal_type: SignalType, current_price: float,
        orderbook_analysis: Dict[str, Any], market_impact: MarketImpact
    ) -> Tuple[float, float]:
        """Calculate stop loss and take profit levels"""
        try:
            # Base risk percentages
            if signal_type in [SignalType.STRONG_BUY, SignalType.STRONG_SELL]:
                sl_percent = 0.02
                tp_percent = 0.06
            elif signal_type in [SignalType.BUY, SignalType.SELL]:
                sl_percent = 0.015
                tp_percent = 0.045
            else:
                sl_percent = 0.025
                tp_percent = 0.05

            # Adjust based on market impact
            sl_percent *= (1 + market_impact.slippage_estimate)
            tp_percent *= (1 + market_impact.slippage_estimate)

            # Calculate levels
            if signal_type in [SignalType.STRONG_BUY, SignalType.BUY, SignalType.REVERSAL_BULLISH]:
                stop_loss = current_price * (1 - sl_percent)
                take_profit = current_price * (1 + tp_percent)
            else:
                stop_loss = current_price * (1 + sl_percent)
                take_profit = current_price * (1 - tp_percent)

            return stop_loss, take_profit

        except Exception as e:
            logger.error(f"Error calculating risk levels: {str(e)}")
            return current_price * 0.98, current_price * 1.02

    def is_range_bound(self, df: pd.DataFrame, market_regime: MarketRegime) -> bool:
        """Determine if market is range-bound"""
        try:
            # Check volatility
            if market_regime.volatility > 0.02:
                return False

            recent_df = df.tail(100)
            price_range = (recent_df['high'].max() - recent_df['low'].min()) / recent_df['close'].mean()

            # Check if price is contained within a narrow range
            if price_range > 0.03:
                return False

            # Check for strong trends
            if market_regime.trend_strength > 0.02:
                return False

            # Check volume stability
            volume_stability = recent_df['volume'].std() / recent_df['volume'].mean() if recent_df['volume'].mean() > 0 else 0.0
            if volume_stability > 0.5:
                return False

            return True

        except Exception as e:
            logger.error(f"Error checking range-bound market: {str(e)}")
            return False

    async def save_signal(self, metrics: SignalMetrics) -> None:
        """Save signal to database"""
        try:
            signal = Signal(
                timestamp=metrics.signal_time,
                chart_timestamp=metrics.chart_timestamp,
                symbol=metrics.symbol,
                price=metrics.current_price,
                price_24h_change=metrics.price_24h_change,
                signal=metrics.signal_type.value,
                stop_loss=metrics.stop_loss,
                take_profit=metrics.take_profit,
                success_rate=metrics.success_rate,
                result=metrics.result,
                analysis_details={
                    'volume_analysis': metrics.volume_analysis,
                    'orderbook_analysis': metrics.orderbook_analysis,
                    'contributing_factors': metrics.contributing_factors,
                    'market_regime': {
                        'volatility': metrics.market_regime.volatility,
                        'trend_strength': metrics.market_regime.trend_strength,
                        'liquidity_score': metrics.market_regime.liquidity_score,
                        'regime_type': metrics.market_regime.regime_type,
                        'confidence': metrics.market_regime.confidence
                    },
                    'market_impact': {
                        'price_impact': metrics.market_impact.price_impact,
                        'slippage_estimate': metrics.market_impact.slippage_estimate,
                        'volume_impact': metrics.market_impact.volume_impact
                    },
                    'reversal_analysis': metrics.reversal_analysis,
                    'institutional_analysis': metrics.institutional_analysis

                }
            )

            await sync_to_async(signal.save)()

        except Exception as e:
            logger.error(f"Error saving signal: {str(e)}")

    async def get_candles_df(self, pair: str, timeframe: str) -> pd.DataFrame:
        """Fetch candlestick data from database"""
        try:
            # Use cache
            cache_key = self.get_cache_key('candles', pair=pair, timeframe=timeframe)
            cached_df = await self.get_cached_data(cache_key)
            if cached_df is not None:
                return cached_df

            klines = await sync_to_async(list)(
                Kline.objects.filter(
                    symbol=pair,
                    timeframe=timeframe
                ).order_by('timestamp')
                .values('timestamp', 'open', 'high', 'low', 'close', 'volume')
            )

            if not klines:
                return pd.DataFrame()

            df = pd.DataFrame(klines)
            df['timestamp'] = pd.to_datetime(df['timestamp'])
            df.set_index('timestamp', inplace=True)
            await self.set_cached_data(cache_key, df, self.cache_timeouts['candles'])
            return df

        except Exception as e:
            logger.error(f"Error fetching candles: {str(e)}")
            return pd.DataFrame()

    async def get_orderbook(self, pair: str) -> Tuple[List, List]:
        """Fetch orderbook data from database"""
        try:
            # Use cache
            cache_key = self.get_cache_key('orderbook', pair=pair)
            cached_orderbook = await self.get_cached_data(cache_key)
            if cached_orderbook is not None:
                return cached_orderbook

            orderbook = await sync_to_async(OrderBook.objects.filter(symbol=pair).first)()
            if not orderbook:
                return [], []

            bids = orderbook.bids
            asks = orderbook.asks
            await self.set_cached_data(cache_key, (bids, asks), self.cache_timeouts['orderbook'])
            return bids, asks

        except Exception as e:
            logger.error(f"Error fetching orderbook: {str(e)}")
            return [], []

    async def run(self):
        """Main method to run the bot for all symbols and timeframes."""
        try:
            symbols = await self.get_symbols()
            tasks = []
            for symbol in symbols:
                for timeframe in self.timeframes:
                    tasks.append(self.generate_signal(symbol, timeframe))
            await asyncio.gather(*tasks)
        except Exception as e:
            logger.error(f"Error running bot: {str(e)}")

    async def get_symbols(self) -> List[str]:
        """Fetch the list of trading pairs (symbols)."""
        trading_pairs = await sync_to_async(list)(TradingPair.objects.values_list('symbol', flat=True))
        return trading_pairs

if __name__ == '__main__':
    bot = EnhancedMarketMicrostructureBot()
    asyncio.run(bot.run())

    Can you analyze the above code to debug it. Tell me more about institutional_presence and institutional_analysis and if they are all implemented as it should. Debug for any incomplete functions or erroneous methods. And lastly, write def calculate_institutional_score function correctly and replace the placeholder with fully implemented logic. If something needs to be modified you need to write the whole method for tthe function.