import asyncio
import logging
import os
from typing import Dict, List, Tuple, Optional, Union, Any
from dataclasses import dataclass
from enum import Enum
import numpy as np
import pandas as pd
from datetime import datetime, timedelta
from collections import defaultdict
import talib
from scipy.stats import norm, pearsonr
from decimal import Decimal
import time
from concurrent.futures import ThreadPoolExecutor
from functools import lru_cache
from statistics import mean, stdev
import warnings

# Django imports
import django
from django.utils import timezone
from asgiref.sync import sync_to_async
from django.db.models import Avg, Max, Min, Q, F
from django.db import transaction
from django.core.cache import cache

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'binancebot.settings')
django.setup()

from .models import Kline, OrderBook, Signal, TradingPair, Trade

# Enhanced logging configuration
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s - %(funcName)s:%(lineno)d',
    handlers=[
        logging.FileHandler('trading_signals.log'),
        logging.StreamHandler()
    ]
)
logger = logging.getLogger(__name__)

class SignalType(Enum):
    STRONG_BUY = "Strong Buy"
    BUY = "Buy"
    NEUTRAL = "Neutral"
    SELL = "Sell"
    STRONG_SELL = "Strong Sell"
    REVERSAL_BULLISH = "Reversal Bullish"
    REVERSAL_BEARISH = "Reversal Bearish"
    NO_SIGNAL = "No Signal"

@dataclass
class MarketRegime:
    volatility: float
    trend_strength: float
    liquidity_score: float
    regime_type: str
    confidence: float
    volume_profile: Dict[str, float]
    market_efficiency: float
    regime_change_probability: float

@dataclass
class MarketImpact:
    price_impact: float
    volume_impact: float
    slippage_estimate: float
    recovery_time: float
    impact_asymmetry: float
    market_resilience: float
    depth_score: float

@dataclass
class SignalMetrics:
    signal_time: datetime
    chart_timestamp: datetime
    symbol: str
    current_price: float
    price_24h_change: float
    signal_type: SignalType
    stop_loss: float
    take_profit: float
    success_rate: float
    result: str
    volume_analysis: Dict
    orderbook_analysis: Dict
    technical_analysis: Dict
    market_regime: MarketRegime
    market_impact: MarketImpact
    reversal_analysis: Optional[Dict] = None
    institutional_analysis: Optional[Dict] = None
    delta_volume_analysis: Optional[Dict] = None
    vwap_analysis: Optional[Dict] = None
    signal_confidence: float = 0.0
    false_signal_probability: float = 0.0

class DynamicThresholds:
    def __init__(self, window_size: int = 100):
        self.window_size = window_size
        self.volume_thresholds: Dict[str, float] = {}
        self.price_thresholds: Dict[str, float] = {}
        self.spread_thresholds: Dict[str, float] = {}
        self.impact_thresholds: Dict[str, float] = {}
        
    async def update_thresholds(self, symbol: str, df: pd.DataFrame, orderbook: Tuple[List, List]) -> None:
        """Update dynamic thresholds based on recent market conditions"""
        try:
            recent_data = df.tail(self.window_size)
            
            # Volume thresholds
            volume_std = recent_data['volume'].std()
            volume_mean = recent_data['volume'].mean()
            self.volume_thresholds[symbol] = {
                'spike': volume_mean + 2 * volume_std,
                'low': volume_mean - volume_std,
                'normal_range': (volume_mean - volume_std, volume_mean + volume_std)
            }
            
            # Price thresholds
            returns = recent_data['close'].pct_change()
            price_std = returns.std()
            self.price_thresholds[symbol] = {
                'volatility': price_std,
                'significant_move': price_std * 2,
                'extreme_move': price_std * 3
            }
            
            # Spread thresholds
            bids, asks = orderbook
            spreads = []
            for i in range(min(10, len(bids), len(asks))):
                bid_price = float(bids[i][0])
                ask_price = float(asks[i][0])
                if bid_price > 0:  # Prevent division by zero
                    spreads.append((ask_price - bid_price) / bid_price)
            
            if spreads:
                spread_mean = mean(spreads)
                spread_std = stdev(spreads) if len(spreads) > 1 else 0
                self.spread_thresholds[symbol] = {
                    'normal': spread_mean,
                    'wide': spread_mean + spread_std,
                    'extreme': spread_mean + 2 * spread_std
                }
            
            # Impact thresholds
            impact_samples = self.calculate_impact_samples(orderbook)
            if impact_samples:
                impact_mean = mean(impact_samples)
                impact_std = stdev(impact_samples) if len(impact_samples) > 1 else 0
                self.impact_thresholds[symbol] = {
                    'low': impact_mean - impact_std,
                    'medium': impact_mean,
                    'high': impact_mean + impact_std
                }
                
        except Exception as e:
            logger.error(f"Error updating dynamic thresholds: {str(e)}")
            self.reset_thresholds(symbol)

    def calculate_impact_samples(self, orderbook: Tuple[List, List]) -> List[float]:
        """Calculate market impact samples for different order sizes"""
        try:
            bids, asks = orderbook
            if not bids or not asks:
                return []
            
            base_size = float(mean([float(bid[1]) for bid in bids[:5]]))
            sample_sizes = [base_size * mult for mult in [0.5, 1, 2, 5]]
            
            impacts = []
            for size in sample_sizes:
                bid_impact = self.calculate_size_impact(bids, size)
                ask_impact = self.calculate_size_impact(asks, size)
                if bid_impact is not None and ask_impact is not None:
                    impacts.append((bid_impact + ask_impact) / 2)
            
            return impacts
            
        except Exception as e:
            logger.error(f"Error calculating impact samples: {str(e)}")
            return []

    @staticmethod
    def calculate_size_impact(orders: List, size: float) -> Optional[float]:
        """Calculate price impact for a given order size"""
        try:
            if not orders:
                return None
                
            cumulative_size = 0
            initial_price = float(orders[0][0])
            
            for price, order_size in orders:
                price = float(price)
                order_size = float(order_size)
                cumulative_size += order_size
                
                if cumulative_size >= size:
                    return abs(price - initial_price) / initial_price if initial_price > 0 else None
                    
            return None
            
        except Exception as e:
            logger.error(f"Error calculating size impact: {str(e)}")
            return None

    def reset_thresholds(self, symbol: str) -> None:
        """Reset thresholds to default values"""
        self.volume_thresholds[symbol] = {
            'spike': 2.0,
            'low': 0.5,
            'normal_range': (0.5, 2.0)
        }
        self.price_thresholds[symbol] = {
            'volatility': 0.02,
            'significant_move': 0.04,
            'extreme_move': 0.06
        }
        self.spread_thresholds[symbol] = {
            'normal': 0.001,
            'wide': 0.002,
            'extreme': 0.005
        }
        self.impact_thresholds[symbol] = {
            'low': 0.001,
            'medium': 0.002,
            'high': 0.005
        }

class EnhancedMarketMicrostructureBot:
    def __init__(self):
        # Core parameters - now using dynamic thresholds
        self.dynamic_thresholds = DynamicThresholds()
        self.short_period = 100
        self.long_period = 500
        self.timeframes = ["1m", "5m", "15m", "1h", "4h", "1d"]
        
        # Enhanced market microstructure parameters
        self.liquidity_levels = [0.1, 0.2, 0.5, 1.0, 2.0, 5.0]
        self.institutional_threshold = 100000.0
        self.max_spread_threshold = 0.002
        self.min_liquidity_requirement = 1000000.0
        
        # Reversal detection parameters
        self.reversal_volume_threshold = 2.0
        self.reversal_price_threshold = 0.02
        self.reversal_confirmation_periods = 3
        self.reversal_momentum_threshold = 0.7
        
        # Performance optimization
        self.max_workers = 4
        self.retry_attempts = 3
        self.backoff_factor = 1.5
        self._setup_cache()
        
        # Statistical validation parameters
        self.confidence_level = 0.95
        self.min_sample_size = 30
        self.correlation_threshold = 0.7
        
        # Error handling
        self.error_registry = defaultdict(int)
        self.max_errors = 5
        self.error_timeout = 300  # seconds

    def _setup_cache(self) -> None:
        """Setup caching parameters"""
        self.cache_timeouts = {
            'candles': 60,
            'orderbook': 5,
            'trades': 10,
            'analysis': 30
        }
        self.cache_keys = defaultdict(dict)

    @staticmethod
    def get_cache_key(prefix: str, **kwargs) -> str:
        """Generate consistent cache keys"""
        sorted_items = sorted(kwargs.items())
        return f"{prefix}:" + ":".join(f"{k}={v}" for k, v in sorted_items)

    async def get_cached_data(self, cache_key: str) -> Optional[Any]:
        """Get data from cache with async support"""
        try:
            return await sync_to_async(cache.get)(cache_key)
        except Exception as e:
            logger.error(f"Cache retrieval error: {str(e)}")
            return None

    async def set_cached_data(self, cache_key: str, data: Any, timeout: int) -> None:
        """Set data in cache with async support"""
        try:
            await sync_to_async(cache.set)(cache_key, data, timeout)
        except Exception as e:
            logger.error(f"Cache setting error: {str(e)}")

    async def handle_error(self, error_key: str, error: Exception) -> bool:
        """Handle errors with exponential backoff and circuit breaker"""
        try:
            self.error_registry[error_key] += 1
            error_count = self.error_registry[error_key]
            
            if error_count >= self.max_errors:
                logger.error(f"Circuit breaker triggered for {error_key}: {str(error)}")
                await self.notify_admin(f"Circuit breaker triggered for {error_key}")
                return False
                
            backoff_time = self.backoff_factor ** (error_count - 1)
            logger.warning(f"Error in {error_key}, attempt {error_count}: {str(error)}")
            await asyncio.sleep(backoff_time)
            
            return True
            
        except Exception as e:
            logger.error(f"Error in error handler: {str(e)}")
            return False

    async def notify_admin(self, message: str) -> None:
        """Notify admin of critical issues"""
        logger.critical(message)
        # Implement your notification logic here (email, Slack, etc.)

    @staticmethod
    def validate_data(data: Any, data_type: str) -> bool:
        """Validate data integrity"""
        try:
            if data_type == 'dataframe':
                return isinstance(data, pd.DataFrame) and not data.empty
            elif data_type == 'orderbook':
                return (isinstance(data, tuple) and len(data) == 2 and
                       all(isinstance(x, list) for x in data))
            elif data_type == 'number':
                return isinstance(data, (int, float)) and not np.isnan(data)
            return False
        except Exception:
            return False

    async def analyze_market_microstructure(self, pair: str, orderbook: Tuple[List, List]) -> Dict:
        """Enhanced market microstructure analysis"""
        try:
            bids, asks = orderbook
            if not bids or not asks:
                return {}

            analysis = {
                'liquidity_metrics': await self.analyze_liquidity_metrics(bids, asks),
                'order_flow_metrics': self.analyze_order_flow(bids, asks),
                'institutional_presence': self.detect_institutional_presence(bids, asks),
                'market_impact': self.calculate_market_impact(bids, asks),
                'microstructure_signals': self.analyze_microstructure_signals(bids, asks),
                'orderbook_pressure': self.calculate_orderbook_pressure(bids, asks)
            }

            return analysis

        except Exception as e:
            logger.error(f"Error in market microstructure analysis: {str(e)}")
            return {}

    async def analyze_liquidity_metrics(self, bids: List, asks: List) -> Dict:
        """Advanced liquidity analysis"""
        metrics = {
            'depth_profile': {},
            'resilience_metrics': {},
            'spread_analysis': {},
            'liquidity_score': 0.0
        }

        try:
            # Spread analysis
            best_bid = float(bids[0][0])
            best_ask = float(asks[0][0])
            spread = (best_ask - best_bid) / best_bid

            # Depth profile at different levels
            for level in self.liquidity_levels:
                depth_range = best_bid * level
                bid_depth = sum(float(bid[1]) for bid in bids if float(bid[0]) >= best_bid - depth_range)
                ask_depth = sum(float(ask[1]) for ask in asks if float(ask[0]) <= best_ask + depth_range)
                
                metrics['depth_profile'][f"level_{int(level*100)}bps"] = {
                    'bid_depth': bid_depth,
                    'ask_depth': ask_depth,
                    'imbalance': (bid_depth - ask_depth) / (bid_depth + ask_depth) if (bid_depth + ask_depth) > 0 else 0
                }

            # Liquidity resilience
            metrics['resilience_metrics'] = self.calculate_resilience_metrics(bids, asks)
            
            # Spread stability
            metrics['spread_analysis'] = {
                'current_spread': spread,
                'spread_stability': self.calculate_spread_stability(bids, asks),
                'effective_spread': self.calculate_effective_spread(bids, asks)
            }

            # Overall liquidity score
            metrics['liquidity_score'] = self.calculate_liquidity_score(metrics)

            return metrics

        except Exception as e:
            logger.error(f"Error in liquidity analysis: {str(e)}")
            return metrics

    def calculate_resilience_metrics(self, bids: List, asks: List) -> Dict:
        """Calculate market resilience metrics"""
        try:
            best_bid = float(bids[0][0])
            best_ask = float(asks[0][0])
            mid_price = (best_bid + best_ask) / 2

            # Calculate order book slope
            bid_slope = self.calculate_order_book_slope(bids, mid_price, side='bid')
            ask_slope = self.calculate_order_book_slope(asks, mid_price, side='ask')

            # Calculate price impact for standard volume
            standard_volume = 100000  # Adjust based on your needs
            bid_impact = self.calculate_price_impact(bids, standard_volume, side='bid')
            ask_impact = self.calculate_price_impact(asks, standard_volume, side='ask')

            return {
                'bid_slope': bid_slope,
                'ask_slope': ask_slope,
                'bid_impact': bid_impact,
                'ask_impact': ask_impact,
                'symmetry': abs(bid_slope - ask_slope) / max(bid_slope, ask_slope),
                'recovery_factor': self.estimate_recovery_factor(bids, asks)
            }

        except Exception as e:
            logger.error(f"Error calculating resilience metrics: {str(e)}")
            return {}

    def calculate_order_book_slope(self, orders: List, mid_price: float, side: str) -> float:
        """Calculate order book slope"""
        try:
            prices = []
            volumes = []
            cumulative_volume = 0

            for price, volume in orders[:10]:  # Use top 10 levels
                price = float(price)
                volume = float(volume)
                prices.append(abs(price - mid_price) / mid_price)  # Normalize prices
                cumulative_volume += volume
                volumes.append(cumulative_volume)

            if not prices or not volumes:
                return 0.0

            # Calculate slope using numpy's polyfit
            slope, _ = np.polyfit(volumes, prices, deg=1)
            return abs(slope)

        except Exception as e:
            logger.error(f"Error calculating order book slope: {str(e)}")
            return 0.0

    def calculate_price_impact(self, orders: List, volume: float, side: str) -> float:
        """Calculate price impact for given volume"""
        try:
            cumulative_volume = 0
            initial_price = float(orders[0][0])

            for price, size in orders:
                price = float(price)
                size = float(size)
                cumulative_volume += size

                if cumulative_volume >= volume:
                    # Calculate weighted average price impact
                    return abs(price - initial_price) / initial_price

            return 1.0  # Maximum impact if order size exceeds available liquidity

        except Exception as e:
            logger.error(f"Error calculating price impact: {str(e)}")
            return 1.0

    def estimate_recovery_factor(self, bids: List, asks: List) -> float:
        """Estimate market recovery factor"""
        try:
            best_bid = float(bids[0][0])
            best_ask = float(asks[0][0])
            mid_price = (best_bid + best_ask) / 2

            # Calculate total available liquidity within 0.5% of mid price
            near_liquidity = sum(float(bid[1]) for bid in bids if float(bid[0]) >= mid_price * 0.995) + \
                           sum(float(ask[1]) for ask in asks if float(ask[0]) <= mid_price * 1.005)

            # Calculate total available liquidity within 2% of mid price
            far_liquidity = sum(float(bid[1]) for bid in bids if float(bid[0]) >= mid_price * 0.98) + \
                          sum(float(ask[1]) for ask in asks if float(ask[0]) <= mid_price * 1.02)

            if far_liquidity == 0:
                return 0.0

            return near_liquidity / far_liquidity

        except Exception as e:
            logger.error(f"Error estimating recovery factor: {str(e)}")
            return 0.0

    def detect_institutional_presence(self, bids: List, asks: List) -> Dict:
        """Detect and analyze institutional order flow"""
        try:
            analysis = {
                'large_orders': {'bids': [], 'asks': []},
                'clustering': {'bids': {}, 'asks': {}},
                'iceberg_probability': 0.0,
                'smart_routing_suggestion': None
            }

            # Detect large orders
            for side, orders in [('bids', bids), ('asks', asks)]:
                large_orders = []
                price_clusters = defaultdict(float)

                for price, size in orders:
                    price, size = float(price), float(size)
                    if size >= float(self.institutional_threshold):
                        large_orders.append({'price': price, 'size': size})
                    
                    # Round price to analyze clustering
                    price_bucket = round(price, 3)
                    price_clusters[price_bucket] += size

                analysis['large_orders'][side] = large_orders
                
                # Analyze clustering
                significant_clusters = {
                    price: volume for price, volume in price_clusters.items()
                    if volume >= float(self.institutional_threshold)
                }
                analysis['clustering'][side] = significant_clusters

            # Detect potential iceberg orders
            analysis['iceberg_probability'] = self.detect_iceberg_orders(bids, asks)

            # Generate smart routing suggestion
            analysis['smart_routing_suggestion'] = self.generate_smart_routing_suggestion(analysis)

            return analysis

        except Exception as e:
            logger.error(f"Error detecting institutional presence: {str(e)}")
            return {}

    def detect_iceberg_orders(self, bids: List, asks: List) -> float:
        """Detect potential iceberg orders"""
        try:
            # Analyze order size patterns
            bid_sizes = [float(bid[1]) for bid in bids[:5]]  # Top 5 levels
            ask_sizes = [float(ask[1]) for ask in asks[:5]]

            # Calculate size variation
            bid_variation = np.std(bid_sizes) / np.mean(bid_sizes) if bid_sizes else 0
            ask_variation = np.std(ask_sizes) / np.mean(ask_sizes) if ask_sizes else 0

            # High variation might indicate iceberg orders
            iceberg_probability = max(bid_variation, ask_variation) / 2

            return min(1.0, iceberg_probability)

        except Exception as e:
            logger.error(f"Error detecting iceberg orders: {str(e)}")
            return 0.0

    def generate_smart_routing_suggestion(self, institutional_analysis: Dict) -> Dict:
        """Generate smart order routing suggestions"""
        try:
            suggestion = {
                'preferred_venue': 'primary',
                'order_type': 'limit',
                'execution_style': 'normal',
                'urgency': 'medium'
            }

            # Analyze institutional presence
            bid_clusters = len(institutional_analysis['clustering']['bids'])
            ask_clusters = len(institutional_analysis['clustering']['asks'])
            large_orders = len(institutional_analysis['large_orders']['bids']) + \
                         len(institutional_analysis['large_orders']['asks'])

            # Adjust suggestion based on market conditions
            if large_orders > 5:
                suggestion['execution_style'] = 'careful'
                suggestion['urgency'] = 'low'
                suggestion['order_type'] = 'iceberg'

            if bid_clusters > 3 or ask_clusters > 3:
                suggestion['execution_style'] = 'aggressive'
                suggestion['urgency'] = 'high'

            if institutional_analysis['iceberg_probability'] > 0.7:
                suggestion['order_type'] = 'peg'
                suggestion['execution_style'] = 'smart'

            return suggestion

        except Exception as e:
            logger.error(f"Error generating smart routing suggestion: {str(e)}")
            return {}

    async def generate_signal(self, pair: str, timeframe: str) -> Tuple[SignalType, SignalMetrics]:
        """Generate comprehensive trading signal"""
        try:
            # Fetch required data
            df = await self.get_candles_df(pair, timeframe)
            if df.empty:
                return SignalType.NO_SIGNAL, None

            orderbook = await self.get_orderbook(pair)
            if not orderbook:
                return SignalType.NO_SIGNAL, None

            # Perform comprehensive analysis
            current_price = float(df['close'].iloc[-1])
            price_24h_ago = float(df['close'].iloc[-1440]) if len(df) > 1440 else float(df['close'].iloc[0])
            price_24h_change = ((current_price - price_24h_ago) / price_24h_ago)
            volume_analysis = await self.analyze_volume_metrics(df)
            orderbook_analysis = await self.analyze_market_microstructure(pair, orderbook)
            
            # Market regime and impact analysis
            market_regime = await self.detect_market_regime(df, orderbook_analysis)
            market_impact = await self.calculate_detailed_market_impact(orderbook)
            
            # Reversal analysis
            reversal_analysis = await self.detect_trend_reversal(df, orderbook, volume_analysis)
            
            # Institutional analysis
            institutional_analysis = await self.analyze_institutional_activity(orderbook, volume_analysis)
            
            # Generate signal based on comprehensive analysis
            signal_type, metrics = await self.determine_signal_type(
                df, orderbook_analysis, volume_analysis, market_regime,
                reversal_analysis, institutional_analysis
            )
            
            if signal_type == SignalType.NO_SIGNAL:
                return signal_type, None
            
            # Calculate risk metrics
            stop_loss, take_profit = await self.calculate_risk_levels(
                signal_type, current_price, orderbook_analysis, market_impact
            )
            
            # Calculate success probability
            success_rate = await self.calculate_success_probability(
                signal_type, volume_analysis, orderbook_analysis,
                market_regime, institutional_analysis
            )
            
            # Prepare signal metrics
            metrics = SignalMetrics(
                signal_time=timezone.now(),
                chart_timestamp=df.index[-1],
                symbol=pair,
                current_price=current_price,
                price_24h_change=price_24h_change * 100,
                signal_type=signal_type,
                stop_loss=stop_loss,
                take_profit=take_profit,
                success_rate=success_rate,
                result="Pending",
                volume_analysis=volume_analysis,
                orderbook_analysis=orderbook_analysis,
                technical_analysis=await self.get_technical_confirmation(df),
                market_regime=market_regime,
                market_impact=market_impact,
                reversal_analysis=reversal_analysis,
                institutional_analysis=institutional_analysis
            )
            
            # Save signal to database
            await self.save_signal(metrics)
            
            return signal_type, metrics
            
    async def analyze_volume_metrics(self, df: pd.DataFrame) -> Dict:
        """Analyze volume patterns and characteristics"""
        try:
            recent_volume = df['volume'].tail(self.short_period)
            volume_sma = recent_volume.mean()
            volume_std = recent_volume.std()
            
            return {
                'current_volume': float(recent_volume.iloc[-1]),
                'volume_sma': float(volume_sma),
                'volume_std': float(volume_std),
                'volume_zscore': float((recent_volume.iloc[-1] - volume_sma) / volume_std),
                'volume_trend': self.calculate_volume_trend(recent_volume),
                'volume_concentration': self.analyze_volume_concentration(df),
                'buying_pressure': self.calculate_buying_pressure(df),
                'volume_profile': self.analyze_volume_profile(df)
            }
        except Exception as e:
            logger.error(f"Error in volume analysis: {str(e)}")
            return {}

    async def determine_signal_type(
        self, df: pd.DataFrame, orderbook_analysis: Dict,
        volume_analysis: Dict, market_regime: MarketRegime,
        reversal_analysis: Dict, institutional_analysis: Dict
    ) -> Tuple[SignalType, Dict]:
        """Determine signal type based on comprehensive analysis"""
        try:
            # Calculate component scores
            volume_score = self.calculate_volume_score(volume_analysis)
            orderbook_score = self.calculate_orderbook_score(orderbook_analysis)
            institutional_score = self.calculate_institutional_score(institutional_analysis)
            
            # Combined weighted score
            total_score = (
                volume_score * 0.4 +
                orderbook_score * 0.4 +
                institutional_score * 0.2
            )
            
            # Check for reversals first
            if reversal_analysis.get('reversal_detected'):
                if reversal_analysis['direction'] == 'bullish':
                    return SignalType.REVERSAL_BULLISH, {}
                return SignalType.REVERSAL_BEARISH, {}
            
            # Check for range-bound market
            if self.is_range_bound(df, market_regime):
                return SignalType.NEUTRAL, {}
            
            # Determine signal based on total score
            if total_score > 0.8:
                return SignalType.STRONG_BUY, {}
            elif total_score > 0.6:
                return SignalType.BUY, {}
            elif total_score < -0.8:
                return SignalType.STRONG_SELL, {}
            elif total_score < -0.6:
                return SignalType.SELL, {}
            
            return SignalType.NO_SIGNAL, {}
            
        except Exception as e:
            logger.error(f"Error determining signal type: {str(e)}")
            return SignalType.NO_SIGNAL, {}

    async def calculate_success_probability(
        self, signal_type: SignalType, volume_analysis: Dict,
        orderbook_analysis: Dict, market_regime: MarketRegime,
        institutional_analysis: Dict
    ) -> float:
        """Calculate probability of signal success"""
        try:
            # Base probability from volume analysis
            volume_probability = min(1.0, volume_analysis.get('volume_zscore', 0) / 3)
            
            # Orderbook confidence
            ob_confidence = orderbook_analysis.get('liquidity_metrics', {}).get('liquidity_score', 0)
            
            # Institutional confidence
            inst_confidence = institutional_analysis.get('iceberg_probability', 0)
            
            # Market regime adjustment
            regime_multiplier = 1.0
            if market_regime.regime_type == "high_volatility":
                regime_multiplier = 0.8
            elif market_regime.regime_type == "low_liquidity":
                regime_multiplier = 0.7
            
            # Combined probability
            base_probability = (
                volume_probability * 0.4 +
                ob_confidence * 0.4 +
                inst_confidence * 0.2
            ) * regime_multiplier
            
            # Adjust based on signal type
            if signal_type in [SignalType.STRONG_BUY, SignalType.STRONG_SELL]:
                base_probability *= 1.2
            elif signal_type in [SignalType.REVERSAL_BULLISH, SignalType.REVERSAL_BEARISH]:
                base_probability *= 0.9
            
            return min(0.95, max(0.05, base_probability))
            
        except Exception as e:
            logger.error(f"Error calculating success probability: {str(e)}")
            return 0.5

    async def calculate_risk_levels(
        self, signal_type: SignalType, current_price: float,
        orderbook_analysis: Dict, market_impact: MarketImpact
    ) -> Tuple[float, float]:
        """Calculate stop loss and take profit levels"""
        try:
            # Get liquidity clusters from orderbook
            liquidity_clusters = orderbook_analysis.get('liquidity_metrics', {}).get('depth_profile', {})
            
            # Base risk percentages
            if signal_type in [SignalType.STRONG_BUY, SignalType.STRONG_SELL]:
                sl_percent = 0.02
                tp_percent = 0.06
            elif signal_type in [SignalType.BUY, SignalType.SELL]:
                sl_percent = 0.015
                tp_percent = 0.045
            else:
                sl_percent = 0.025
                tp_percent = 0.05
            
            # Adjust based on market impact
            sl_percent *= (1 + market_impact.slippage_estimate)
            tp_percent *= (1 + market_impact.slippage_estimate)
            
            # Calculate levels
            if signal_type in [SignalType.STRONG_BUY, SignalType.BUY, SignalType.REVERSAL_BULLISH]:
                stop_loss = current_price * (1 - sl_percent)
                take_profit = current_price * (1 + tp_percent)
            else:
                stop_loss = current_price * (1 + sl_percent)
                take_profit = current_price * (1 - tp_percent)
            
            return stop_loss, take_profit
            
        except Exception as e:
            logger.error(f"Error calculating risk levels: {str(e)}")
            return current_price * 0.98, current_price * 1.02

    async def detect_market_regime(self, df: pd.DataFrame, orderbook_analysis: Dict) -> MarketRegime:
        """Detect current market regime"""
        try:
            # Calculate volatility
            returns = df['close'].pct_change()
            volatility = returns.std() * np.sqrt(len(df))
            
            # Calculate trend strength
            price_sma = df['close'].rolling(window=20).mean()
            trend_strength = abs(df['close'].iloc[-1] - price_sma.iloc[-1]) / price_sma.iloc[-1]
            
            # Get liquidity score
            liquidity_score = orderbook_analysis.get('liquidity_metrics', {}).get('liquidity_score', 0)
            
            # Determine regime type
            if volatility > 0.02 and liquidity_score < 0.5:
                regime_type = "high_volatility_low_liquidity"
                confidence = 0.8
            elif volatility > 0.02:
                regime_type = "high_volatility"
                confidence = 0.85
            elif liquidity_score < 0.5:
                regime_type = "low_liquidity"
                confidence = 0.75
            elif trend_strength > 0.02:
                regime_type = "trending"
                confidence = 0.9
            else:
                regime_type = "normal"
                confidence = 0.7
            
            return MarketRegime(
                volatility=float(volatility),
                trend_strength=float(trend_strength),
                liquidity_score=float(liquidity_score),
                regime_type=regime_type,
                confidence=confidence
            )
            
        except Exception as e:
            logger.error(f"Error detecting market regime: {str(e)}")
            return MarketRegime(0, 0, 0, "unknown", 0)

    async def detect_trend_reversal(
        self, df: pd.DataFrame, orderbook: Tuple[List, List],
        volume_analysis: Dict
    ) -> Dict:
        """Detect potential trend reversals"""
        try:
            bids, asks = orderbook
            current_price = float(df['close'].iloc[-1])
            
            analysis = {
                'reversal_detected': False,
                'direction': None,
                'confidence': 0,
                'supporting_factors': []
            }
            
            # Volume surge check
            volume_zscore = volume_analysis.get('volume_zscore', 0)
            if abs(volume_zscore) > self.reversal_volume_threshold:
                analysis['supporting_factors'].append('volume_surge')
            
            # Orderbook imbalance check
            bid_volume = sum(float(bid[1]) for bid in bids[:10])
            ask_volume = sum(float(ask[1]) for ask in asks[:10])
            ob_imbalance = (bid_volume - ask_volume) / (bid_volume + ask_volume)
            
            if abs(ob_imbalance) > self.ob_imbalance_threshold:
                analysis['supporting_factors'].append('orderbook_imbalance')
            
            # Price movement check
            price_change = (df['close'].pct_change()
                          .rolling(window=self.reversal_confirmation_periods)
                          .sum()
                          .iloc[-1])
            
            if abs(price_change) > self.reversal_price_threshold:
                analysis['supporting_factors'].append('price_movement')
            
            # Determine reversal
            if len(analysis['supporting_factors']) >= 2:
                analysis['reversal_detected'] = True
                analysis['direction'] = 'bullish' if ob_imbalance > 0 else 'bearish'
                analysis['confidence'] = min(0.9, 0.6 + len(analysis['supporting_factors']) * 0.1)
            
            return analysis
            
        except Exception as e:
            logger.error(f"Error detecting trend reversal: {str(e)}")
            return {'reversal_detected': False}

    async def save_signal(self, metrics: SignalMetrics) -> None:
        """Save signal to database"""
        try:
            signal = Signal(
                timestamp=metrics.signal_time,
                chart_timestamp=metrics.chart_timestamp,
                symbol=metrics.symbol,
                price=metrics.current_price,
              
                analysis_details={
                    'volume_analysis': metrics.volume_analysis,
                    'orderbook_analysis': metrics.orderbook_analysis,
                    'technical_analysis': metrics.technical_analysis,
                    'market_regime': {
                        'volatility': metrics.market_regime.volatility,
                        'trend_strength': metrics.market_regime.trend_strength,
                        'liquidity_score': metrics.market_regime.liquidity_score,
                        'regime_type': metrics.market_regime.regime_type,
                        'confidence': metrics.market_regime.confidence
                    },
                    'market_impact': {
                        'price_impact': metrics.market_impact.price_impact,
                        'slippage_estimate': metrics.market_impact.slippage_estimate,

                    },

                }
            )
            
            await sync_to_async(signal.save)()
            
        except Exception as e:
            logger.error(f"Error saving signal: {str(e)}")

    async def get_candles_df(self, pair: str, timeframe: str) -> pd.DataFrame:
        """Fetch candlestick data from database"""
        try:
            klines = await sync_to_async(list)(
                Kline.objects.filter(
                    symbol=pair,
                    timeframe=timeframe
                ).order_by('timestamp')
                .values('timestamp', 'open', 'high', 'low', 'close', 'volume')
            )
            
            if not klines:
                return pd.DataFrame()
            
            df = pd.DataFrame(klines)
            df.set_index('timestamp', inplace=True)
            return df
            
        except Exception as e:
            logger.error(f"Error fetching candles: {str(e)}")
            return pd.DataFrame()

    async def get_orderbook(self, pair: str) -> Tuple[List, List]:
        """Fetch orderbook data from database"""
        try:
            orderbook = await sync_to_async(OrderBook.objects.filter(symbol=pair).first)()
            if not orderbook:
                return [], []
            
            return orderbook.bids, orderbook.asks
            
        except Exception as e:
            logger.error(f"Error fetching orderbook: {str(e)}")
            return [], []

    async def calculate_detailed_market_impact(self, orderbook: Tuple[List, List]) -> MarketImpact:
        """Calculate detailed market impact metrics"""
        try:
            bids, asks = orderbook
            if not bids or not asks:
                return MarketImpact(0, 0, 0, 0)

            # Calculate immediate price impact for standard size
            standard_size = 10000  # Adjust based on your needs
            bid_impact = self.calculate_price_impact(bids, standard_size, 'bid')
            ask_impact = self.calculate_price_impact(asks, standard_size, 'ask')
            price_impact = (bid_impact + ask_impact) / 2

            # Calculate volume impact
            bid_volume = sum(float(bid[1]) for bid in bids[:10])
            ask_volume = sum(float(ask[1]) for ask in asks[:10])
            total_volume = bid_volume + ask_volume
            volume_impact = standard_size / total_volume if total_volume > 0 else 1

            # Estimate slippage
            slippage_estimate = price_impact * (1 + volume_impact)

            # Estimate recovery time based on volume and impact
            recovery_time = self.estimate_recovery_time(slippage_estimate, volume_impact)

            return MarketImpact(
                price_impact=float(price_impact),
                volume_impact=float(volume_impact),
                slippage_estimate=float(slippage_estimate),
                recovery_time=float(recovery_time)
            )

        except Exception as e:
            logger.error(f"Error calculating market impact: {str(e)}")
            return MarketImpact(0, 0, 0, 0)

    def estimate_recovery_time(self, slippage: float, volume_impact: float) -> float:
        """Estimate market recovery time in minutes"""
        try:
            base_recovery = 5  # Base recovery time in minutes
            impact_multiplier = 1 + (slippage * 10)  # Higher slippage = longer recovery
            volume_multiplier = 1 + (volume_impact * 5)  # Higher volume impact = longer recovery
            
            return base_recovery * impact_multiplier * volume_multiplier
            
        except Exception as e:
            logger.error(f"Error estimating recovery time: {str(e)}")
            return 5.0

    def calculate_volume_trend(self, volumes: pd.Series) -> str:
        """Analyze volume trend"""
        try:
            volume_sma = volumes.rolling(window=20).mean()
            current_vol = volumes.iloc[-1]
            avg_vol = volume_sma.iloc[-1]
            
            if current_vol > avg_vol * 2:
                return "strong_increasing"
            elif current_vol > avg_vol * 1.5:
                return "increasing"
            elif current_vol < avg_vol * 0.5:
                return "strong_decreasing"
            elif current_vol < avg_vol * 0.75:
                return "decreasing"
            else:
                return "stable"
                
        except Exception as e:
            logger.error(f"Error calculating volume trend: {str(e)}")
            return "unknown"

    def analyze_volume_concentration(self, df: pd.DataFrame) -> Dict:
        """Analyze volume concentration patterns"""
        try:
            recent_df = df.tail(100)
            volume_price_levels = pd.cut(recent_df['close'], bins=10)
            volume_concentration = recent_df.groupby(volume_price_levels)['volume'].sum()
            
            return {
                'high_volume_levels': volume_concentration.nlargest(3).index.astype(str).tolist(),
                'concentration_ratio': float(volume_concentration.max() / volume_concentration.sum()),
                'distribution': volume_concentration.to_dict()
            }
            
        except Exception as e:
            logger.error(f"Error analyzing volume concentration: {str(e)}")
            return {}

    def calculate_buying_pressure(self, df: pd.DataFrame) -> float:
        """Calculate buying pressure based on volume and price movement"""
        try:
            recent_df = df.tail(20)
            up_volume = recent_df[recent_df['close'] > recent_df['open']]['volume'].sum()
            down_volume = recent_df[recent_df['close'] < recent_df['open']]['volume'].sum()
            total_volume = up_volume + down_volume
            
            if total_volume == 0:
                return 0.5
                
            return float(up_volume / total_volume)
            
        except Exception as e:
            logger.error(f"Error calculating buying pressure: {str(e)}")
            return 0.5

    def analyze_volume_profile(self, df: pd.DataFrame) -> Dict:
        """Analyze volume profile characteristics"""
        try:
            recent_df = df.tail(100)
            
            profile = {
                'avg_volume': float(recent_df['volume'].mean()),
                'vol_std': float(recent_df['volume'].std()),
                'volume_pods': {},
                'volume_trend_strength': 0.0
            }
            
            # Calculate volume Points of Distribution
            price_levels = pd.qcut(recent_df['close'], q=10)
            volume_pods = recent_df.groupby(price_levels)['volume'].agg(['sum', 'count'])
            profile['volume_pods'] = volume_pods.to_dict()
            
            # Calculate volume trend strength
            volume_sma = recent_df['volume'].rolling(window=20).mean()
            volume_trend = recent_df['volume'].iloc[-1] / volume_sma.iloc[-1] if not volume_sma.empty else 1
            profile['volume_trend_strength'] = float(volume_trend)
            
            return profile
            
        except Exception as e:
            logger.error(f"Error analyzing volume profile: {str(e)}")
            return {}

    def is_range_bound(self, df: pd.DataFrame, market_regime: MarketRegime) -> bool:
        """Determine if market is range-bound"""
        try:
            # Check volatility
            if market_regime.volatility > 0.02:
                return False
                
            recent_df = df.tail(100)
            price_range = (recent_df['high'].max() - recent_df['low'].min()) / recent_df['close'].mean()
            
            # Check if price is contained within a narrow range
            if price_range > 0.03:
                return False
                
            # Check for strong trends
            if market_regime.trend_strength > 0.02:
                return False
                
            # Check volume stability
            volume_stability = recent_df['volume'].std() / recent_df['volume'].mean()
            if volume_stability > 0.5:
                return False
                
            return True
            
        except Exception as e:
            logger.error(f"Error checking range-bound market: {str(e)}")
            return False

    async def get_technical_confirmation(self, df: pd.DataFrame) -> Dict:
        """Get minimal technical analysis confirmation"""
        try:
            recent_close = df['close'].values
            
            # Calculate basic momentum
            roc = (recent_close[-1] - recent_close[-20]) / recent_close[-20]
            
            # Calculate basic volatility
            returns = pd.Series(recent_close).pct_change()
            volatility = returns.std()
            
            return {
                'momentum': float(roc),
                'volatility': float(volatility),
                'price_trend': 'up' if roc > 0 else 'down',
                'volatility_regime': 'high' if volatility > 0.02 else 'low'
            }
            
        except Exception as e:
            logger.error(f"Error getting technical confirmation: {str(e)}")
            return {}

    def calculate_spread_stability(self, bids: List, asks: List) -> float:
        """Calculate spread stability metric"""
        try:
            spreads = []
            for i in range(min(10, len(bids), len(asks))):
                bid_price = float(bids[i][0])
                ask_price = float(asks[i][0])
                spreads.append((ask_price - bid_price) / bid_price)
                
            return float(np.std(spreads)) if spreads else 1.0
            
        except Exception as e:
            logger.error(f"Error calculating spread stability: {str(e)}")
            return 1.0

    def calculate_effective_spread(self, bids: List, asks: List) -> float:
        """Calculate effective spread considering market impact"""
        try:
            if not bids or not asks:
                return 0.0
                
            top_bid = float(bids[0][0])
            top_ask = float(asks[0][0])
            mid_price = (top_bid + top_ask) / 2
            
            return float((top_ask - top_bid) / mid_price)
            
        except Exception as e:
            logger.error(f"Error calculating effective spread: {str(e)}")
            return 0.0

In above code, record the what analysis generated the signal. For instance, this is orderbook. So when saving in database, I want to know this signal or the details in additional information was generated by orderbook. This is because if I add technical analysis and machine learning models, I should be able to tell what signals belong to what analysis to tell which one is accurate and which is not.
I want you to improve it by; Optimize Parameters and Thresholds: Adjust the parameters used in your analyses using systematic parameter optimization techniques and consider using dynamic thresholds that adapt to changing market conditions. Implement robust retry mechanisms and fallback options. Incorporate statistical tests to validate the significance of your findings in volume spikes, order book imbalances, and other analyses. Examine the correlation between different indicators to avoid redundant or conflicting signals. Implement filters to reduce market noise, such as smoothing techniques. Analyze changes in bid/ask sizes and prices over time, not just static snapshots. Look for patterns like aggressive order placement or large order cancellations. Go beyond the top levels of the order book. Analyze the cumulative volume at different price levels to understand liquidity distribution. Incorporate trade data (price, volume, direction) to identify aggressive buying/selling activity. Use techniques like grid search or Bayesian optimization to find optimal parameter values for your specific trading pair and timeframe. Add any other features that would increase the accuracy of the bot. Error Handling & Data Validation, Missing true volume profile analysis (VWAP levels), No delta volume calculation (buy vs sell volume at each price level). false signal filtering.
I also want you to debug the code and resolve issues in it and any missing features. Some of the issues are
Type Errors: Mixing Decimal and float can lead to unexpected behavior. Be consistent, use float all trou. The calculation for price_24h_ago in generate_signal assumes 1-minute data. If the timeframe is different, the index -1440 will be incorrect. You should use time-based filtering instead of index-based filtering. Wrap more code blocks in try...except blocks to handle potential errors gracefully. Cache Implementation: The cache is not being used. If you want to implement caching, you need to add logic to store and retrieve data from the cache. 
analyze_institutional_activity Method Missing, calculate_institutional_score Method Missing:. Incorrect or Missing Return Statements eg async def generate_signal Missing Return Metrics. Possible Division by Zero Errors, f initial_price is zero, dividing by it will cause a division by zero error, Issue: If volume_sma.iloc[-1] is zero, dividing by zero will occur. When calling synchronous methods from asynchronous code, you use sync_to_async. Ensure that all database access is properly managed to prevent blocking the event loop. Potential KeyErrors and AttributeErrors.  Ensure that all methods consistently return the expected data types, especially when exceptions occur. Enhance logging messages to include more context, which will aid in debugging. Debug the code to identify more issues and resolve them. 
provide a more accurate approach of generating trading signals. Enhance every aspect of the code, use advanced functionalities that make the bot robust. To help in monitoring performance and improving it, I want to know as much information as possible. When generating and saving signals, I want to know the time the bot generated that specific signal(if it runs now it records the time now or if it was 10.30 in the morning), the timestamp (records the time on the price chart when signal was generated), symbol (the crypto pair being analyzed and for which the signal is generated), price(current real time price of the crypto), 24h change (how much the coin or token has gained or lost in the last 24 hours in %), signal (if is strong buy, buy, sell, strong sell), Stop Loss (Price at whichI should exit the trade to avoid further losses. It can be slightly above resistance level for short trades or below support for long trades), Take profit (Price at which I should take profits and exit trade. It can be at the resistance level for longs and at support for shorts), Success rate (This is the probability of the signal to result in a win or profit expressed in percentage based on the strength of the signal and factors supporting that signal), and finally Results (If real time price reaches stop loss first before it reaches take profit level the result is Lost, if it reaches the take profit level first before stop loss it is Won and if it is still swinging in between the result is Pending). I also want to see more details on how the bot reached to that signal, example what in order_book indicated buy or sell , what is it about volume and other factors that led to saying Strong buy or Strong Sell. Make the analysis comprehensive to determine if it is a strong buy, or just Buy and same for Sell. I also want Neutral Signals for symbols that are likely to swing between two price points over extended periods of time. Do not just conclude that all symbols which are not buy or sell are Neutral, if the future is unpredictable it should be no signal. Only those that will remain in a specific price brackets are Neutral. NOTE: PRICE DATA FOR ALL SYMBOLS IN '1m', '5m', '15m', '1h' TIMEFRAMES IS SAVED IN DJANGO ORM DATABASE SO YOU NEED TO QUERY IT FROM DJANO MODELS AND CONVERT IT TO PD DATAFRAME AND USE IT FOR ANALYSIS. GENERATE SIGNALS FOR THE DIFFERENT TIMEFRAMES ABOVE FOR ALL SYMBOLS CONCURRENTLY USING ASYNCIO AND ALSO COMBINE TO CONFIRM. USE ADVANCED FEATURES THAT WOULD MAKE ORDERBOOK ACCURACY CLOSE TO 80%. ONLY FOCUS ON ORDERBOOK, DO NOT INTRODUCE MACHINE LEARNING MODELS OR TECHNICAL INDICATORS.