import numpy as np
import pandas as pd
from sklearn.feature_selection import SelectKBest, f_regression
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split, TimeSeriesSplit, RandomizedSearchCV
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.svm import SVR
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Dense, Dropout
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping
from statsmodels.tsa.arima.model import ARIMA
from fbprophet import Prophet
from talib import add_all_ta_features
from add_all_ta_features import MACD
from ta.volatility import BollingerBands
from ta.trend import MACD, EMAIndicator
from ta.momentum import RSIIndicator
import talib
from scipy.stats import randint, uniform
from joblib import Parallel, delayed

class AdvancedCryptoTradingBot:
    def __init__(self):
        self.models = {
            'random_forest': RandomForestRegressor(random_state=42),
            'svm': SVR(),
            'xgboost': XGBRegressor(random_state=42),
            'lightgbm': LGBMRegressor(random_state=42),
            'gradient_boosting': GradientBoostingRegressor(random_state=42),
            'lstm': None,
            'arima': None,
            'prophet': Prophet(),
            'garch': arch_model(y=None, vol='GARCH', p=1, q=1)
        }
        self.scaler = StandardScaler()
        self.best_params = {}
        self.feature_importances = {}
        self.model_weights = {model: 1/len(self.models) for model in self.models}
        self.model_performance = {model: [] for model in self.models}
    
    def preprocess_data(self, data):
        data = self.handle_missing_data(data)
        data = self.handle_outliers(data)
        data = self.add_technical_indicators(data)
        data = self.add_fourier_features(data)
        data = self.add_lagged_features(data)
        data = self.add_market_regime_features(data)
        data = self.add_sentiment_features(data)
        data = self.add_multi_timeframe_features(data)
        data.dropna(inplace=True)
        return data
    def add_technical_indicators(self, data):
        # Add all TA features
        data = add_all_ta_features(data, open="open", high="high", low="low", close="close", volume="volume")
        
        # Add custom indicators
        bb = BollingerBands(data['close'])
        data['bb_high'] = bb.bollinger_hband()
        data['bb_low'] = bb.bollinger_lband()
        
        macd = MACD(data['close'])
        data['macd'] = macd.macd()
        data['macd_signal'] = macd.macd_signal()
        
        ema = EMAIndicator(data['close'])
        data['ema'] = ema.ema_indicator()
        
        rsi = RSIIndicator(data['close'])
        data['rsi'] = rsi.rsi()
        
        # Add TALib indicators
        data['adx'] = talib.ADX(data['high'], data['low'], data['close'])
        data['cci'] = talib.CCI(data['high'], data['low'], data['close'])
        data['mfi'] = talib.MFI(data['high'], data['low'], data['close'], data['volume'])
        
        return data
    
    def add_fourier_features(self, data):
        # Add Fourier features to capture cyclical patterns
        for period in [5, 10, 22]:  # represent weekly, biweekly, and monthly cycles
            for order in range(1, 3):  # use sine and cosine
                data[f'fourier_sin_{period}_{order}'] = np.sin(2 * np.pi * order * data.index.dayofyear / period)
                data[f'fourier_cos_{period}_{order}'] = np.cos(2 * np.pi * order * data.index.dayofyear / period)
        return data
    
    def add_lagged_features(self, data):
        # Add lagged features
        for lag in [1, 3, 5, 7, 14, 30]:
            data[f'close_lag_{lag}'] = data['close'].shift(lag)
            data[f'volume_lag_{lag}'] = data['volume'].shift(lag)
        return data
    
    def add_market_regime_features(self, data):
        # Identify market regimes using volatility
        returns = data['close'].pct_change()
        volatility = returns.rolling(window=22).std()
        data['market_regime'] = pd.cut(volatility, bins=3, labels=['low', 'medium', 'high'])
        data = pd.get_dummies(data, columns=['market_regime'])
        return data
      
    def get_param_distributions(self, model_name):
        if model_name == 'random_forest':
            return {
                'n_estimators': randint(100, 500),
                'max_depth': randint(5, 30),
                'min_samples_split': randint(2, 20),
                'min_samples_leaf': randint(1, 10)
            }
        elif model_name == 'svm':
            return {
                'C': uniform(0.1, 10),
                'gamma': uniform(0.01, 1),
                'kernel': ['rbf', 'poly']
            }
        elif model_name == 'xgboost':
            return {
                'n_estimators': randint(100, 500),
                'max_depth': randint(3, 10),
                'learning_rate': uniform(0.01, 0.3),
                'subsample': uniform(0.6, 0.4),
                'colsample_bytree': uniform(0.6, 0.4)
            }
        elif model_name == 'lightgbm':
            return {
                'n_estimators': randint(100, 500),
                'max_depth': randint(3, 10),
                'learning_rate': uniform(0.01, 0.3),
                'subsample': uniform(0.6, 0.4),
                'colsample_bytree': uniform(0.6, 0.4)
            }
        elif model_name == 'gradient_boosting':
            return {
                'n_estimators': randint(100, 500),
                'max_depth': randint(3, 10),
                'learning_rate': uniform(0.01, 0.3),
                'subsample': uniform(0.6, 0.4),
                'min_samples_split': randint(2, 20),
                'min_samples_leaf': randint(1, 10)
            }
    
    def handle_missing_data(self, data):
        # Simple forward fill for missing data
        return data.ffill()
    
    def handle_outliers(self, data):
        # Use IQR method to remove outliers
        Q1 = data.quantile(0.25)
        Q3 = data.quantile(0.75)
        IQR = Q3 - Q1
        lower_bound = Q1 - 1.5 * IQR
        upper_bound = Q3 + 1.5 * IQR
        return data[(data >= lower_bound) & (data <= upper_bound)]
    
    def add_sentiment_features(self, data):
        # Placeholder for sentiment analysis
        # In a real implementation, you would fetch sentiment data from an API
        data['sentiment'] = np.random.uniform(-1, 1, len(data))
        return data
    
    def add_multi_timeframe_features(self, data):
        # Add features from multiple timeframes
        for column in ['close', 'volume']:
            for timeframe in ['1H', '4H', '1D']:
                resampled = data[column].resample(timeframe).last()
                data[f'{column}_change_{timeframe}'] = resampled.pct_change()
        return data
    
    def prepare_features(self, data):
        features = data.drop(['open', 'high', 'low', 'close', 'volume'], axis=1)
        target = data['close'].shift(-1)  # Next day's closing price
        
        # Feature selection
        selector = SelectKBest(f_regression, k=50)
        features = pd.DataFrame(selector.fit_transform(features, target), columns=features.columns[selector.get_support()])
        
        return features, target
    
    def train_models(self, data):
        features, target = self.prepare_features(data)
        X_train, X_test, y_train, y_test = train_test_split(features, target, test_size=0.2, shuffle=False)
        
        X_train_scaled = self.scaler.fit_transform(X_train)
        X_test_scaled = self.scaler.transform(X_test)
        
        for model_name in ['random_forest', 'svm', 'xgboost', 'lightgbm', 'gradient_boosting']:
            self.tune_and_train(model_name, X_train_scaled, y_train, X_test_scaled, y_test)
        
        self.models['lstm'] = self.train_lstm(X_train_scaled, y_train, X_test_scaled, y_test)
        self.models['arima'] = self.train_arima(data['close'])
        
        prophet_data = pd.DataFrame({'ds': data.index, 'y': data['close']})
        self.models['prophet'].fit(prophet_data)
        
        self.models['garch'].fit(data['close'])
        
        self.calculate_feature_importances(features.columns)
    
    def tune_and_train(self, model_name, X_train, y_train, X_test, y_test):
        print(f"Tuning and training {model_name}...")
        param_distributions = self.get_param_distributions(model_name)
        
        tscv = TimeSeriesSplit(n_splits=5)
        
        search = RandomizedSearchCV(
            self.models[model_name],
            param_distributions=param_distributions,
            n_iter=50,
            cv=tscv,
            random_state=42,
            n_jobs=-1,
            scoring='neg_mean_squared_error'
        )
        
        search.fit(X_train, y_train)
        self.models[model_name] = search.best_estimator_
        self.best_params[model_name] = search.best_params_
        
        # Evaluate performance
        y_pred = self.models[model_name].predict(X_test)
        mse = mean_squared_error(y_test, y_pred)
        self.model_performance[model_name].append(mse)
        
        print(f"Best parameters for {model_name}: {search.best_params_}")
        print(f"MSE for {model_name}: {mse}")
    
    def train_lstm(self, X_train, y_train, X_test, y_test):
        model = Sequential([
            LSTM(100, activation='relu', input_shape=(X_train.shape[1], 1), return_sequences=True),
            Dropout(0.2),
            LSTM(50, activation='relu'),
            Dropout(0.2),
            Dense(1)
        ])
        model.compile(optimizer=Adam(learning_rate=0.001), loss='mse')
        
        X_train_reshaped = X_train.reshape((X_train.shape[0], X_train.shape[1], 1))
        X_test_reshaped = X_test.reshape((X_test.shape[0], X_test.shape[1], 1))
        
        early_stopping = EarlyStopping(patience=10, restore_best_weights=True)
        
        tscv = TimeSeriesSplit(n_splits=5)
        for train_index, val_index in tscv.split(X_train_reshaped):
            X_train_split, X_val_split = X_train_reshaped[train_index], X_train_reshaped[val_index]
            y_train_split, y_val_split = y_train.iloc[train_index], y_train.iloc[val_index]
            
            model.fit(
                X_train_split, y_train_split,
                epochs=100,
                batch_size=32,
                validation_data=(X_val_split, y_val_split),
                callbacks=[early_stopping],
                verbose=0
            )
        
        # Evaluate performance
        y_pred = model.predict(X_test_reshaped)
        mse = mean_squared_error(y_test, y_pred)
        self.model_performance['lstm'].append(mse)
        print(f"MSE for LSTM: {mse}")
        
        return model
    
    def train_arima(self, time_series):
        # Use auto_arima for optimal order selection
        model = auto_arima(time_series, start_p=1, start_q=1, max_p=5, max_q=5, m=7,
                           start_P=0, seasonal=True, d=1, D=1, trace=True,
                           error_action='ignore', suppress_warnings=True, stepwise=True)
        
        # Evaluate performance
        y_pred = model.predict(n_periods=len(time_series))
        mse = mean_squared_error(time_series, y_pred)
        self.model_performance['arima'].append(mse)
        print(f"MSE for ARIMA: {mse}")
        
        return model
    
    def update_model_weights(self):
        total_performance = sum(sum(perf) for perf in self.model_performance.values())
        for model in self.models:
            if self.model_performance[model]:
                self.model_weights[model] = 1 - (sum(self.model_performance[model]) / total_performance)
        
        # Normalize weights
        total_weight = sum(self.model_weights.values())
        for model in self.model_weights:
            self.model_weights[model] /= total_weight
    
    def generate_signals(self, latest_data):
        features, _ = self.prepare_features(latest_data)
        scaled_features = self.scaler.transform(features)
        
        signals = {}
        confidences = {}
        
        for model_name in ['random_forest', 'svm', 'xgboost', 'lightgbm', 'gradient_boosting']:
            prediction = self.models[model_name].predict(scaled_features)
            signals[model_name] = 1 if prediction[-1] > latest_data['close'].iloc[-1] else -1
            confidences[model_name] = abs(prediction[-1] - latest_data['close'].iloc[-1]) / latest_data['close'].iloc[-1]
        
        lstm_features = scaled_features.reshape((scaled_features.shape[0], scaled_features.shape[1], 1))
        lstm_prediction = self.models['lstm'].predict(lstm_features)
        signals['lstm'] = 1 if lstm_prediction[-1][0] > latest_data['close'].iloc[-1] else -1
        confidences['lstm'] = abs(lstm_prediction[-1][0] - latest_data['close'].iloc[-1]) / latest_data['close'].iloc[-1]
        
        arima_forecast = self.models['arima'].forecast(steps=1)
        signals['arima'] = 1 if arima_forecast[0] > latest_data['close'].iloc[-1] else -1
        confidences['arima'] = abs(arima_forecast[0] - latest_data['close'].iloc[-1]) / latest_data['close'].iloc[-1]
        
        prophet_forecast = self.models['prophet'].predict(pd.DataFrame({'ds': [latest_data.index[-1] + pd.Timedelta(days=1)]}))
        signals['prophet'] = 1 if prophet_forecast['yhat'].iloc[0] > latest_data['close'].iloc[-1] else -1
        confidences['prophet'] = abs(prophet_forecast['yhat'].iloc[0] - latest_data['close'].iloc[-1]) / latest_data['close'].iloc[-1]
        
        garch_forecast = self.models['garch'].forecast(horizon=1)
        signals['garch'] = 1 if garch_forecast.mean.iloc[-1] > latest_data['close'].iloc[-1] else -1
        confidences['garch'] = abs(garch_forecast.mean.iloc[-1] - latest_data['close'].iloc[-1]) / latest_data['close'].iloc[-1]
        
        return signals, confidences
    
    def ensemble_decision(self, signals, confidences):
        weighted_sum = sum(self.model_weights[model] * signal * confidences[model] for model, signal in signals.items())
        
        if weighted_sum > 0.1:
            return 1  # Strong buy signal
        elif weighted_sum < -0.1:
            return -1  # Strong sell signal
        else:
            return 0  # Hold signal
    
    def calculate_position_size(self, confidence, balance, current_price):
        # Calculate position size based on Kelly Criterion
        win_rate = 0.5 + confidence  # Assume win rate is correlated with confidence
        kelly_fraction = win_rate - ((1 - win_rate) / (self.risk_reward_ratio - 1))
        kelly_fraction = max(0, min(kelly_fraction, 0.2))  # Limit position size to 20% of balance
        return (balance * kelly_fraction) / current_price
    
    def backtest(self, data, initial_balance=10000, risk_reward_ratio=2):
        self.risk_reward_ratio = risk_reward_ratio
        balance = initial_balance
        position = 0
        trades = []
        daily_returns = []
        
        for i in range(len(data) - 30):  # Use 30-day lookback for signal generation
            lookback_data = data.iloc[i:i+30]
            signals, confidences = self.generate_signals(lookback_data)
            decision = self.ensemble_decision(signals, confidences)
            
            price = data['close'].iloc[i+30]
            confidence = abs(sum(self.model_weights[model] * signals[model] * confidences[model] for model in signals))
            
            if decision == 1 and position <= 0:  # Buy signal
                shares_to_buy = self.calculate_position_size(confidence, balance, price)
                position += shares_to_buy
                balance -= shares_to_buy * price * (1 + self.transaction_cost)
                trades.append(('buy', price, shares_to_buy))
            elif decision == -1 and position > 0:  # Sell signal
                balance += position * price * (1 - self.transaction_cost)
                trades.append(('sell', price, position))
                position = 0
            
            # Apply stop-loss and take-profit
            if position > 0:
                if price <= trades[-1][1] * (1 - self.stop_loss):
                    balance += position * price * (1 - self.transaction_cost)
                    trades.append(('stop_loss', price, position))
                    position = 0
                elif price >= trades[-1][1] * (1 + self.take_profit):
                    balance += position * price * (1 - self.transaction_cost)
                    trades.append(('take_profit', price, position))
                    position = 0
            
            daily_return = (balance + position * price) / initial_balance - 1
            daily_returns.append(daily_return)
        
        # Close any remaining position
        if position > 0:
            balance += position * data['close'].iloc[-1] * (1 - self.transaction_cost)
        
        sharpe_ratio = np.sqrt(252) * np.mean(daily_returns) / np.std(daily_returns)
        max_drawdown = np.max(np.maximum.accumulate(daily_returns) - daily_returns)
        
        return balance, trades, sharpe_ratio, max_drawdown
    
    def parallel_backtest(self, data, num_simulations=100):
        results = Parallel(n_jobs=-1)(
            delayed(self.backtest)(data) for _ in range(num_simulations)
        )
        final_balances = [r[0] for r in results]
        sharpe_ratios = [r[2] for r in results]
        max_drawdowns = [r[3] for r in results]
        
        return {
            'mean_balance': np.mean(final_balances),
            'std_balance': np.std(final_balances),
            'mean_sharpe': np.mean(sharpe_ratios),
            'mean_max_drawdown': np.mean(max_drawdowns)
        }
    
    def explain_decision(self, signals, confidences):
        explanation = "Model predictions:\n"
        for model, signal in signals.items():
            explanation += f"{model}: {'Buy' if signal == 1 else 'Sell'} (confidence: {confidences[model]:.2f})\n"
        
        explanation += f"\nModel weights:\n"
        for model, weight in self.model_weights.items():
            explanation += f"{model}: {weight:.2f}\n"
        
        weighted_sum = sum(self.model_weights[model] * signal * confidences[model] for model, signal in signals.items())
        explanation += f"\nWeighted sum: {weighted_sum:.2f}"
        
        if weighted_sum > 0.1:
            explanation += "\nFinal decision: Strong buy"
        elif weighted_sum < -0.1:
            explanation += "\nFinal decision: Strong sell"
        else:
            explanation += "\nFinal decision: Hold"
        
        return explanation
    
    def calculate_shap_values(self, data):
        features, _ = self.prepare_features(data)
        explainer = shap.TreeExplainer(self.models['random_forest'])
        shap_values = explainer.shap_values(features)
        return shap_values, features.columns
    
    def plot_feature_importance(self, data):
        shap_values, feature_names = self.calculate_shap_values(data)
        shap.summary_plot(shap_values, features=data[feature_names], feature_names=feature_names)
    
    def adapt_to_market_regime(self, data):
        # Identify market regime
        returns = data['close'].pct_change()
        volatility = returns.rolling(window=22).std()
        regime = pd.cut(volatility, bins=3, labels=['low', 'medium', 'high'])
        
        # Train separate models for each regime
        for r in ['low', 'medium', 'high']:
            regime_data = data[regime == r]
            self.train_models(regime_data)
        
        self.current_regime = regime.iloc[-1]
    
    def online_update(self, new_data):
        # Periodically retrain models with new data
        self.train_models(new_data)
        
        # Update model weights based on recent performance
        self.update_model_weights()
        
        # Adapt to new market regime
        self.adapt_to_market_regime(new_data)

# Example usage
if __name__ == "__main__":
    # Load your data here
    data = pd.read_csv('your_crypto_data.csv', parse_dates=['timestamp'], index_col='timestamp')
    
    bot = AdvancedCryptoTradingBot()
    
    # Preprocess and train the models
    preprocessed_data = bot.preprocess_data(data)
    bot.train_models(preprocessed_data)
    
    # Generate signals for the latest data point
    latest_data = preprocessed_data.iloc[-30:]  # Use more data points for better signal generation
    signals, confidences = bot.generate_signals(latest_data)
    final_decision = bot.ensemble_decision(signals, confidences)
    
    # print("Individual model signals:", signals)
    print("Final decision:", "Strong Buy" if final_decision == 1 else "Strong Sell" if final_decision == -1 else "Hold")
    
    # Explain the decision
    explanation = bot.explain_decision(signals, confidences)
    print(explanation)
    
    # Perform backtesting
    backtest_results = bot.parallel_backtest(preprocessed_data)
    print("Backtesting results:")
    print(f"Mean final balance: ${backtest_results['mean_balance']:.2f}")
    print(f"Standard deviation of final balance: ${backtest_results['std_balance']:.2f}")
    print(f"Mean Sharpe ratio: {backtest_results['mean_sharpe']:.2f}")
    print(f"Mean maximum drawdown: {backtest_results['mean_max_drawdown']:.2f}")
    
    # Plot feature importance
    bot.plot_feature_importance(preprocessed_data)
    
    # Periodically update the model with new data
    new_data = fetch_new_data()  # Function to fetch new data
    bot.online_update(new_data)


    Can you modify my code with; Advanced Feature Engineering: * Incorporate on-chain metrics (e.g., active addresses, transaction volume) * Use natural language processing on social 
    media and news to better capture sentiment * Implement cross-asset features (e.g., correlations with other cryptocurrencies, stock markets, commodities) Deep Learning Enhancements: 
    * Experiment with more advanced architectures like Transformer models or Graph Neural Networks I dont have enough Computational Resources to train models so focus on pretrained models only
    * Implement attention mechanisms in your LSTM model * Try hybrid models that combine CNN layers for pattern recognition with LSTM layers for time series analysis Multi-timeframe Analysis: 
    * Develop separate models for different timeframes (e.g., 5m, 15m, 1h) and generate signals for the different timeframes. * Create a meta-model that combines predictions from various timeframes, 
    Save all signals from a single timeframe and combined predictions. * Increase Data Granularity: Use high-frequency data (e.g., minute-level data) to capture more detailed market movements. Reinforcement Learning: 
    * Implement a Deep Q-Network (DQN) or Proximal Policy Optimization (PPO) algorithm to learn optimal trading strategies * Advanced Ensemble Techniques: * Implement stacking or blending of models instead of simple weighted averaging 
    * Use meta-learning algorithms to dynamically adjust model weights * Continual Learning: * Implement techniques to allow your models to adapt to new patterns without forgetting old ones * Causal Inference: 
    * Implement causal discovery algorithms to better understand the factors truly driving price changes After making the changes please analyze the code to identify strengths and weaknesses, and modify the code again to tackle the weaknesses and send me the final updated code.
    Avoid Overfitting Risk: 
    Ensure Hyperparameter Optimization is comprehensive for all models.
    include robust error handling for potential issues like data unavailability or
    Ensure consistent debuggin