import numpy as np
import pandas as pd
from sklearn.feature_selection import SelectKBest, mutual_info_regression
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split, TimeSeriesSplit
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.svm import SVR
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Dense, Dropout, Attention, Conv1D
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping
from statsmodels.tsa.arima.model import ARIMA
from prophet import Prophet
import talib
from scipy.stats import randint, uniform
from joblib import Parallel, delayed
import asyncio
import aiohttp
import shap
from typing import Dict, List, Tuple
import logging
from sentiment import get_sentiment_analysis
from orderbook import get_volume_orderbook_analysis
from db_utils import save_signals, save_ml_results
from report_generator import generate_html_report

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

class AdvancedCryptoTradingBot:
    def __init__(self):
        self.models = {
            'random_forest': RandomForestRegressor(random_state=42),
            'svm': SVR(),
            'xgboost': XGBRegressor(random_state=42),
            'lightgbm': LGBMRegressor(random_state=42),
            'gradient_boosting': GradientBoostingRegressor(random_state=42),
            'lstm': None,
            'hybrid_cnn_lstm': None,
            'arima': None,
            'prophet': Prophet(),
        }
        self.scaler = StandardScaler()
        self.best_params = {}
        self.feature_importances = {}
        self.model_weights = {model: 1/len(self.models) for model in self.models}
        self.model_performance = {model: [] for model in self.models}
        self.timeframes = ['5m', '15m', '1h', '4h', '1d']
        self.symbols = []  # List of USDT trading pairs
        self.session = None
    
    async def initialize(self):
        self.session = aiohttp.ClientSession()
        self.symbols = await self.fetch_usdt_trading_pairs()
    
    async def fetch_usdt_trading_pairs(self) -> List[str]:
        # Implement fetching of USDT trading pairs from Binance API
        # This is a placeholder and needs to be implemented
        pass
    
    async def fetch_data(self, symbol: str, timeframe: str) -> pd.DataFrame:
        # Implement data fetching from your Django ORM database
        # This is a placeholder and needs to be implemented
        pass
    
    def preprocess_data(self, data: pd.DataFrame) -> pd.DataFrame:
        data = self.handle_missing_data(data)
        data = self.handle_outliers(data)
        data = self.add_technical_indicators(data)
        data = self.add_on_chain_metrics(data)
        data = self.add_cross_asset_features(data)
        data = self.add_fourier_features(data)
        data = self.add_lagged_features(data)
        data = self.add_market_regime_features(data)
        data = self.add_multi_timeframe_features(data)
        data.dropna(inplace=True)
        return data
    
    def add_technical_indicators(self, data: pd.DataFrame) -> pd.DataFrame:
        # Add all TA features
        for col in ['open', 'high', 'low', 'close', 'volume']:
            if col not in data.columns:
                logger.warning(f"Column {col} not found in data. Skipping technical indicators.")
                return data
        
        try:
            data['rsi'] = talib.RSI(data['close'])
            data['macd'], data['macd_signal'], _ = talib.MACD(data['close'])
            data['bbands_upper'], data['bbands_middle'], data['bbands_lower'] = talib.BBANDS(data['close'])
            data['adx'] = talib.ADX(data['high'], data['low'], data['close'])
            data['cci'] = talib.CCI(data['high'], data['low'], data['close'])
            data['mfi'] = talib.MFI(data['high'], data['low'], data['close'], data['volume'])
        except Exception as e:
            logger.error(f"Error calculating technical indicators: {str(e)}")
        
        return data
    
    def add_on_chain_metrics(self, data: pd.DataFrame) -> pd.DataFrame:
        # Placeholder for adding on-chain metrics
        # This needs to be implemented based on your data sources
        return data
    
    def add_cross_asset_features(self, data: pd.DataFrame) -> pd.DataFrame:
        # Placeholder for adding cross-asset features
        # This needs to be implemented based on your data sources
        return data
    
    def add_fourier_features(self, data: pd.DataFrame) -> pd.DataFrame:
        for period in [5, 10, 22]:  # represent weekly, biweekly, and monthly cycles
            for order in range(1, 3):  # use sine and cosine
                data[f'fourier_sin_{period}_{order}'] = np.sin(2 * np.pi * order * data.index.dayofyear / period)
                data[f'fourier_cos_{period}_{order}'] = np.cos(2 * np.pi * order * data.index.dayofyear / period)
        return data
    
    def add_lagged_features(self, data: pd.DataFrame) -> pd.DataFrame:
        for lag in [1, 3, 5, 7, 14, 30]:
            data[f'close_lag_{lag}'] = data['close'].shift(lag)
            data[f'volume_lag_{lag}'] = data['volume'].shift(lag)
        return data
    
    def add_market_regime_features(self, data: pd.DataFrame) -> pd.DataFrame:
        returns = data['close'].pct_change()
        volatility = returns.rolling(window=22).std()
        data['market_regime'] = pd.cut(volatility, bins=3, labels=['low', 'medium', 'high'])
        data = pd.get_dummies(data, columns=['market_regime'])
        return data
    
    def add_multi_timeframe_features(self, data: pd.DataFrame) -> pd.DataFrame:
        for column in ['close', 'volume']:
            for timeframe in ['1H', '4H', '1D']:
                resampled = data[column].resample(timeframe).last()
                data[f'{column}_change_{timeframe}'] = resampled.pct_change()
        return data
    
    def prepare_features(self, data: pd.DataFrame) -> Tuple[pd.DataFrame, pd.Series]:
        features = data.drop(['open', 'high', 'low', 'close', 'volume'], axis=1)
        target = data['close'].shift(-1)  # Next day's closing price
        
        # Feature selection
        selector = SelectKBest(mutual_info_regression, k=50)
        features = pd.DataFrame(selector.fit_transform(features, target), columns=features.columns[selector.get_support()])
        
        return features, target
    
    async def train_models(self, data: pd.DataFrame):
        features, target = self.prepare_features(data)
        X_train, X_test, y_train, y_test = train_test_split(features, target, test_size=0.2, shuffle=False)
        
        X_train_scaled = self.scaler.fit_transform(X_train)
        X_test_scaled = self.scaler.transform(X_test)
        
        tasks = []
        for model_name in ['random_forest', 'svm', 'xgboost', 'lightgbm', 'gradient_boosting']:
            tasks.append(self.tune_and_train(model_name, X_train_scaled, y_train, X_test_scaled, y_test))
        
        tasks.append(self.train_lstm(X_train_scaled, y_train, X_test_scaled, y_test))
        tasks.append(self.train_hybrid_cnn_lstm(X_train_scaled, y_train, X_test_scaled, y_test))
        tasks.append(self.train_arima(data['close']))
        
        prophet_data = pd.DataFrame({'ds': data.index, 'y': data['close']})
        tasks.append(self.train_prophet(prophet_data))
        
        await asyncio.gather(*tasks)
        
        self.calculate_feature_importances(features.columns)
    
    async def tune_and_train(self, model_name: str, X_train: np.ndarray, y_train: pd.Series, X_test: np.ndarray, y_test: pd.Series):
        logger.info(f"Tuning and training {model_name}...")
        param_distributions = self.get_param_distributions(model_name)
        
        tscv = TimeSeriesSplit(n_splits=5)
        
        search = RandomizedSearchCV(
            self.models[model_name],
            param_distributions=param_distributions,
            n_iter=50,
            cv=tscv,
            random_state=42,
            n_jobs=-1,
            scoring='neg_mean_squared_error'
        )
        
        search.fit(X_train, y_train)
        self.models[model_name] = search.best_estimator_
        self.best_params[model_name] = search.best_params_
        
        # Evaluate performance
        y_pred = self.models[model_name].predict(X_test)
        mse = mean_squared_error(y_test, y_pred)
        self.model_performance[model_name].append(mse)
        
        logger.info(f"Best parameters for {model_name}: {search.best_params_}")
        logger.info(f"MSE for {model_name}: {mse}")
    
    async def train_lstm(self, X_train: np.ndarray, y_train: pd.Series, X_test: np.ndarray, y_test: pd.Series):
        model = Sequential([
            LSTM(100, activation='relu', input_shape=(X_train.shape[1], 1), return_sequences=True),
            Attention(),
            Dropout(0.2),
            LSTM(50, activation='relu'),
            Dropout(0.2),
            Dense(1)
        ])
        model.compile(optimizer=Adam(learning_rate=0.001), loss='mse')
        
        X_train_reshaped = X_train.reshape((X_train.shape[0], X_train.shape[1], 1))
        X_test_reshaped = X_test.reshape((X_test.shape[0], X_test.shape[1], 1))
        
        early_stopping = EarlyStopping(patience=10, restore_best_weights=True)
        
        model.fit(
            X_train_reshaped, y_train,
            epochs=100,
            batch_size=32,
            validation_split=0.2,
            callbacks=[early_stopping],
            verbose=0
        )
        
        # Evaluate performance
        y_pred = model.predict(X_test_reshaped)
        mse = mean_squared_error(y_test, y_pred)
        self.model_performance['lstm'].append(mse)
        logger.info(f"MSE for LSTM: {mse}")
        
        self.models['lstm'] = model
    
    async def train_hybrid_cnn_lstm(self, X_train: np.ndarray, y_train: pd.Series, X_test: np.ndarray, y_test: pd.Series):
        model = Sequential([
            Conv1D(filters=64, kernel_size=3, activation='relu', input_shape=(X_train.shape[1], 1)),
            Conv1D(filters=64, kernel_size=3, activation='relu'),
            LSTM(50, activation='relu', return_sequences=True),
            Attention(),
            LSTM(50, activation='relu'),
            Dense(1)
        ])
        model.compile(optimizer=Adam(learning_rate=0.001), loss='mse')
        
        X_train_reshaped = X_train.reshape((X_train.shape[0], X_train.shape[1], 1))
        X_test_reshaped = X_test.reshape((X_test.shape[0], X_test.shape[1], 1))
        
        early_stopping = EarlyStopping(patience=10, restore_best_weights=True)
        
        model.fit(
            X_train_reshaped, y_train,
            epochs=100,
            batch_size=32,
            validation_split=0.2,
            callbacks=[early_stopping],
            verbose=0
        )
        
        # Evaluate performance
        y_pred = model.predict(X_test_reshaped)
        mse = mean_squared_error(y_test, y_pred)
        self.model_performance['hybrid_cnn_lstm'].append(mse)
        logger.info(f"MSE for Hybrid CNN-LSTM: {mse}")
        
        self.models['hybrid_cnn_lstm'] = model
    
    async def train_arima(self, time_series: pd.Series):
        model = ARIMA(time_series, order=(5,1,0))
        fitted = model.fit()
        
        # Evaluate performance
        y_pred = fitted.forecast(steps=len(time_series))
        mse = mean_squared_error(time_series, y_pred)
        self.model_performance['arima'].append(mse)
        logger.info(f"MSE for ARIMA: {mse}")
        
        self.models['arima'] = fitted
    
    async def train_prophet(self, prophet_data: pd.DataFrame):
        model = Prophet()
        model.fit(prophet_data)
        
        future = model.make_future_dataframe(periods=len(prophet_data))
        forecast = model.predict(future)
        
        # Evaluate performance
        y_pred = forecast['yhat'][-len(prophet_data):]
        mse = mean_squared_error(prophet_data['y'], y_pred)
        self.model_performance['prophet'].append(mse)
        logger.info(f"MSE for Prophet: {mse}")
        
        self.models['prophet'] = model
    
    def update_model_weights(self):
        total_performance = sum(sum(perf) for perf in self.model_performance.values())
        for model in self.models:
            if self.model_performance[model]:
                self.model_weights[model]
    def update_model_weights(self):
        total_performance = sum(sum(perf) for perf in self.model_performance.values())
        for model in self.models:
            if self.model_performance[model]:
                self.model_weights[model] = 1 - (sum(self.model_performance[model]) / total_performance)
        
        # Normalize weights
        total_weight = sum(self.model_weights.values())
        for model in self.model_weights:
            self.model_weights[model] /= total_weight

    async def generate_signals(self, latest_data: pd.DataFrame) -> Tuple[Dict[str, int], Dict[str, float]]:
        features, _ = self.prepare_features(latest_data)
        scaled_features = self.scaler.transform(features)
        
        signals = {}
        confidences = {}
        
        for model_name in ['random_forest', 'svm', 'xgboost', 'lightgbm', 'gradient_boosting']:
            prediction = self.models[model_name].predict(scaled_features)
            signals[model_name] = 1 if prediction[-1] > latest_data['close'].iloc[-1] else -1
            confidences[model_name] = abs(prediction[-1] - latest_data['close'].iloc[-1]) / latest_data['close'].iloc[-1]
        
        lstm_features = scaled_features.reshape((scaled_features.shape[0], scaled_features.shape[1], 1))
        lstm_prediction = self.models['lstm'].predict(lstm_features)
        signals['lstm'] = 1 if lstm_prediction[-1][0] > latest_data['close'].iloc[-1] else -1
        confidences['lstm'] = abs(lstm_prediction[-1][0] - latest_data['close'].iloc[-1]) / latest_data['close'].iloc[-1]
        
        hybrid_prediction = self.models['hybrid_cnn_lstm'].predict(lstm_features)
        signals['hybrid_cnn_lstm'] = 1 if hybrid_prediction[-1][0] > latest_data['close'].iloc[-1] else -1
        confidences['hybrid_cnn_lstm'] = abs(hybrid_prediction[-1][0] - latest_data['close'].iloc[-1]) / latest_data['close'].iloc[-1]
        
        arima_forecast = self.models['arima'].forecast(steps=1)
        signals['arima'] = 1 if arima_forecast[0] > latest_data['close'].iloc[-1] else -1
        confidences['arima'] = abs(arima_forecast[0] - latest_data['close'].iloc[-1]) / latest_data['close'].iloc[-1]
        
        prophet_forecast = self.models['prophet'].predict(pd.DataFrame({'ds': [latest_data.index[-1] + pd.Timedelta(days=1)]}))
        signals['prophet'] = 1 if prophet_forecast['yhat'].iloc[0] > latest_data['close'].iloc[-1] else -1
        confidences['prophet'] = abs(prophet_forecast['yhat'].iloc[0] - latest_data['close'].iloc[-1]) / latest_data['close'].iloc[-1]
        
        return signals, confidences

    def ensemble_decision(self, signals: Dict[str, int], confidences: Dict[str, float]) -> int:
        weighted_sum = sum(self.model_weights[model] * signal * confidences[model] for model, signal in signals.items())
        
        if weighted_sum > 0.1:
            return 1  # Strong buy signal
        elif weighted_sum < -0.1:
            return -1  # Strong sell signal
        else:
            return 0  # Hold signal

    def calculate_position_size(self, confidence: float, balance: float, current_price: float) -> float:
        # Calculate position size based on Kelly Criterion
        win_rate = 0.5 + confidence  # Assume win rate is correlated with confidence
        kelly_fraction = win_rate - ((1 - win_rate) / (self.risk_reward_ratio - 1))
        kelly_fraction = max(0, min(kelly_fraction, 0.2))  # Limit position size to 20% of balance
        return (balance * kelly_fraction) / current_price

    async def backtest(self, data: pd.DataFrame, initial_balance: float = 10000, risk_reward_ratio: float = 2) -> Tuple[float, List[Tuple[str, float, float]], float, float]:
        self.risk_reward_ratio = risk_reward_ratio
        balance = initial_balance
        position = 0
        trades = []
        daily_returns = []
        
        for i in range(len(data) - 30):  # Use 30-day lookback for signal generation
            lookback_data = data.iloc[i:i+30]
            signals, confidences = await self.generate_signals(lookback_data)
            decision = self.ensemble_decision(signals, confidences)
            
            price = data['close'].iloc[i+30]
            confidence = abs(sum(self.model_weights[model] * signals[model] * confidences[model] for model in signals))
            
            if decision == 1 and position <= 0:  # Buy signal
                shares_to_buy = self.calculate_position_size(confidence, balance, price)
                position += shares_to_buy
                balance -= shares_to_buy * price * (1 + self.transaction_cost)
                trades.append(('buy', price, shares_to_buy))
            elif decision == -1 and position > 0:  # Sell signal
                balance += position * price * (1 - self.transaction_cost)
                trades.append(('sell', price, position))
                position = 0
            
            daily_return = (balance + position * price) / initial_balance - 1
            daily_returns.append(daily_return)
        
        # Close any remaining position
        if position > 0:
            balance += position * data['close'].iloc[-1] * (1 - self.transaction_cost)
        
        sharpe_ratio = np.sqrt(252) * np.mean(daily_returns) / np.std(daily_returns)
        max_drawdown = np.max(np.maximum.accumulate(daily_returns) - daily_returns)
        
        return balance, trades, sharpe_ratio, max_drawdown

    async def parallel_backtest(self, data: pd.DataFrame, num_simulations: int = 100) -> Dict[str, float]:
        tasks = [self.backtest(data) for _ in range(num_simulations)]
        results = await asyncio.gather(*tasks)
        
        final_balances = [r[0] for r in results]
        sharpe_ratios = [r[2] for r in results]
        max_drawdowns = [r[3] for r in results]
        
        return {
            'mean_balance': np.mean(final_balances),
            'std_balance': np.std(final_balances),
            'mean_sharpe': np.mean(sharpe_ratios),
            'mean_max_drawdown': np.mean(max_drawdowns)
        }

    def explain_decision(self, signals: Dict[str, int], confidences: Dict[str, float]) -> str:
        explanation = "Model predictions:\n"
        for model, signal in signals.items():
            explanation += f"{model}: {'Buy' if signal == 1 else 'Sell'} (confidence: {confidences[model]:.2f})\n"
        
        explanation += f"\nModel weights:\n"
        for model, weight in self.model_weights.items():
            explanation += f"{model}: {weight:.2f}\n"
        
        weighted_sum = sum(self.model_weights[model] * signal * confidences[model] for model, signal in signals.items())
        explanation += f"\nWeighted sum: {weighted_sum:.2f}"
        
        if weighted_sum > 0.1:
            explanation += "\nFinal decision: Strong buy"
        elif weighted_sum < -0.1:
            explanation += "\nFinal decision: Strong sell"
        else:
            explanation += "\nFinal decision: Hold"
        
        return explanation

    def calculate_shap_values(self, data: pd.DataFrame) -> Tuple[np.ndarray, List[str]]:
        features, _ = self.prepare_features(data)
        explainer = shap.TreeExplainer(self.models['random_forest'])
        shap_values = explainer.shap_values(features)
        return shap_values, features.columns.tolist()

    def plot_feature_importance(self, data: pd.DataFrame):
        shap_values, feature_names = self.calculate_shap_values(data)
        shap.summary_plot(shap_values, features=data[feature_names], feature_names=feature_names)

    async def adapt_to_market_regime(self, data: pd.DataFrame):
        # Identify market regime
        returns = data['close'].pct_change()
        volatility = returns.rolling(window=22).std()
        regime = pd.cut(volatility, bins=3, labels=['low', 'medium', 'high'])
        
        # Train separate models for each regime
        regime_tasks = []
        for r in ['low', 'medium', 'high']:
            regime_data = data[regime == r]
            regime_tasks.append(self.train_models(regime_data))
        
        await asyncio.gather(*regime_tasks)
        
        self.current_regime = regime.iloc[-1]

    async def online_update(self, new_data: pd.DataFrame):
        # Periodically retrain models with new data
        await self.train_models(new_data)
        
        # Update model weights based on recent performance
        self.update_model_weights()
        
        # Adapt to new market regime
        await self.adapt_to_market_regime(new_data)

    async def run(self):
        await self.initialize()
        
        for symbol in self.symbols:
            for timeframe in self.timeframes:
                data = await self.fetch_data(symbol, timeframe)
                preprocessed_data = self.preprocess_data(data)
                await self.train_models(preprocessed_data)
                
                signals, confidences = await self.generate_signals(preprocessed_data.iloc[-30:])
                final_decision = self.ensemble_decision(signals, confidences)
                
                explanation = self.explain_decision(signals, confidences)
                
                # Save signals and results
                await save_signals(symbol, timeframe, final_decision, signals, confidences)
                await save_ml_results(symbol, timeframe, explanation, self.model_performance, self.model_weights)
                
                # Generate and save HTML report
                report_html = generate_html_report(symbol, timeframe, final_decision, explanation, self.model_performance, self.model_weights)
                # Save report_html to file or database
                
                # Perform backtesting
                backtest_results = await self.parallel_backtest(preprocessed_data)
                logger.info(f"Backtesting results for {symbol} {timeframe}:")
                logger.info(f"Mean final balance: ${backtest_results['mean_balance']:.2f}")
                logger.info(f"Standard deviation of final balance: ${backtest_results['std_balance']:.2f}")
                logger.info(f"Mean Sharpe ratio: {backtest_results['mean_sharpe']:.2f}")
                logger.info(f"Mean maximum drawdown: {backtest_results['mean_max_drawdown']:.2f}")
                
                # Plot feature importance
                self.plot_feature_importance(preprocessed_data)
        
        # Close the aiohttp session
        await self.session.close()

if __name__ == "__main__":
    bot = AdvancedCryptoTradingBot()
    asyncio.run(bot.run())
In the code above, it says mean_squared_error is deprecated, which other method can I replace it with to achieve same results or even better.