import numpy as np
import pandas as pd
from sklearn.feature_selection import SelectKBest, f_regression
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split, TimeSeriesSplit, RandomizedSearchCV
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.svm import SVR
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Dense, Dropout
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping
from statsmodels.tsa.arima.model import ARIMA
from fbprophet import Prophet
from talib import add_all_ta_features
from add_all_ta_features import MACD
from ta.volatility import BollingerBands
from ta.trend import MACD, EMAIndicator
from ta.momentum import RSIIndicator
import talib
from scipy.stats import randint, uniform
from joblib import Parallel, delayed

class AdvancedCryptoTradingBot:
    def __init__(self):
        self.models = {
            'random_forest': RandomForestRegressor(random_state=42),
            'svm': SVR(),
            'xgboost': XGBRegressor(random_state=42),
            'lightgbm': LGBMRegressor(random_state=42),
            'gradient_boosting': GradientBoostingRegressor(random_state=42),
            'lstm': None,
            'arima': None,
            'prophet': Prophet(),
            'garch': arch_model(y=None, vol='GARCH', p=1, q=1)
        }
        self.scaler = StandardScaler()
        self.best_params = {}
        self.feature_importances = {}
        self.model_weights = {model: 1/len(self.models) for model in self.models}
        self.model_performance = {model: [] for model in self.models}
    
    def preprocess_data(self, data):
        data = self.handle_missing_data(data)
        data = self.handle_outliers(data)
        data = self.add_technical_indicators(data)
        data = self.add_fourier_features(data)
        data = self.add_lagged_features(data)
        data = self.add_market_regime_features(data)
        data = self.add_sentiment_features(data)
        data = self.add_multi_timeframe_features(data)
        data.dropna(inplace=True)
        return data
    def add_technical_indicators(self, data):
        # Add all TA features
        data = add_all_ta_features(data, open="open", high="high", low="low", close="close", volume="volume")
        
        # Add custom indicators
        bb = BollingerBands(data['close'])
        data['bb_high'] = bb.bollinger_hband()
        data['bb_low'] = bb.bollinger_lband()
        
        macd = MACD(data['close'])
        data['macd'] = macd.macd()
        data['macd_signal'] = macd.macd_signal()
        
        ema = EMAIndicator(data['close'])
        data['ema'] = ema.ema_indicator()
        
        rsi = RSIIndicator(data['close'])
        data['rsi'] = rsi.rsi()
        
        # Add TALib indicators
        data['adx'] = talib.ADX(data['high'], data['low'], data['close'])
        data['cci'] = talib.CCI(data['high'], data['low'], data['close'])
        data['mfi'] = talib.MFI(data['high'], data['low'], data['close'], data['volume'])
        
        return data
    
    def add_fourier_features(self, data):
        # Add Fourier features to capture cyclical patterns
        for period in [5, 10, 22]:  # represent weekly, biweekly, and monthly cycles
            for order in range(1, 3):  # use sine and cosine
                data[f'fourier_sin_{period}_{order}'] = np.sin(2 * np.pi * order * data.index.dayofyear / period)
                data[f'fourier_cos_{period}_{order}'] = np.cos(2 * np.pi * order * data.index.dayofyear / period)
        return data
    
    def add_lagged_features(self, data):
        # Add lagged features
        for lag in [1, 3, 5, 7, 14, 30]:
            data[f'close_lag_{lag}'] = data['close'].shift(lag)
            data[f'volume_lag_{lag}'] = data['volume'].shift(lag)
        return data
    
    def add_market_regime_features(self, data):
        # Identify market regimes using volatility
        returns = data['close'].pct_change()
        volatility = returns.rolling(window=22).std()
        data['market_regime'] = pd.cut(volatility, bins=3, labels=['low', 'medium', 'high'])
        data = pd.get_dummies(data, columns=['market_regime'])
        return data
      
    def get_param_distributions(self, model_name):
        if model_name == 'random_forest':
            return {
                'n_estimators': randint(100, 500),
                'max_depth': randint(5, 30),
                'min_samples_split': randint(2, 20),
                'min_samples_leaf': randint(1, 10)
            }
        elif model_name == 'svm':
            return {
                'C': uniform(0.1, 10),
                'gamma': uniform(0.01, 1),
                'kernel': ['rbf', 'poly']
            }
        elif model_name == 'xgboost':
            return {
                'n_estimators': randint(100, 500),
                'max_depth': randint(3, 10),
                'learning_rate': uniform(0.01, 0.3),
                'subsample': uniform(0.6, 0.4),
                'colsample_bytree': uniform(0.6, 0.4)
            }
        elif model_name == 'lightgbm':
            return {
                'n_estimators': randint(100, 500),
                'max_depth': randint(3, 10),
                'learning_rate': uniform(0.01, 0.3),
                'subsample': uniform(0.6, 0.4),
                'colsample_bytree': uniform(0.6, 0.4)
            }
        elif model_name == 'gradient_boosting':
            return {
                'n_estimators': randint(100, 500),
                'max_depth': randint(3, 10),
                'learning_rate': uniform(0.01, 0.3),
                'subsample': uniform(0.6, 0.4),
                'min_samples_split': randint(2, 20),
                'min_samples_leaf': randint(1, 10)
            }
    
    def handle_missing_data(self, data):
        # Simple forward fill for missing data
        return data.ffill()
    
    def handle_outliers(self, data):
        # Use IQR method to remove outliers
        Q1 = data.quantile(0.25)
        Q3 = data.quantile(0.75)
        IQR = Q3 - Q1
        lower_bound = Q1 - 1.5 * IQR
        upper_bound = Q3 + 1.5 * IQR
        return data[(data >= lower_bound) & (data <= upper_bound)]
    
    def add_sentiment_features(self, data):
        # Placeholder for sentiment analysis
        # In a real implementation, you would fetch sentiment data from an API
        data['sentiment'] = np.random.uniform(-1, 1, len(data))
        return data
    
    def add_multi_timeframe_features(self, data):
        # Add features from multiple timeframes
        for column in ['close', 'volume']:
            for timeframe in ['1H', '4H', '1D']:
                resampled = data[column].resample(timeframe).last()
                data[f'{column}_change_{timeframe}'] = resampled.pct_change()
        return data
    
    def prepare_features(self, data):
        features = data.drop(['open', 'high', 'low', 'close', 'volume'], axis=1)
        target = data['close'].shift(-1)  # Next day's closing price
        
        # Feature selection
        selector = SelectKBest(f_regression, k=50)
        features = pd.DataFrame(selector.fit_transform(features, target), columns=features.columns[selector.get_support()])
        
        return features, target
    
    def train_models(self, data):
        features, target = self.prepare_features(data)
        X_train, X_test, y_train, y_test = train_test_split(features, target, test_size=0.2, shuffle=False)
        
        X_train_scaled = self.scaler.fit_transform(X_train)
        X_test_scaled = self.scaler.transform(X_test)
        
        for model_name in ['random_forest', 'svm', 'xgboost', 'lightgbm', 'gradient_boosting']:
            self.tune_and_train(model_name, X_train_scaled, y_train, X_test_scaled, y_test)
        
        self.models['lstm'] = self.train_lstm(X_train_scaled, y_train, X_test_scaled, y_test)
        self.models['arima'] = self.train_arima(data['close'])
        
        prophet_data = pd.DataFrame({'ds': data.index, 'y': data['close']})
        self.models['prophet'].fit(prophet_data)
        
        self.models['garch'].fit(data['close'])
        
        self.calculate_feature_importances(features.columns)
    
    def tune_and_train(self, model_name, X_train, y_train, X_test, y_test):
        print(f"Tuning and training {model_name}...")
        param_distributions = self.get_param_distributions(model_name)
        
        tscv = TimeSeriesSplit(n_splits=5)
        
        search = RandomizedSearchCV(
            self.models[model_name],
            param_distributions=param_distributions,
            n_iter=50,
            cv=tscv,
            random_state=42,
            n_jobs=-1,
            scoring='neg_mean_squared_error'
        )
        
        search.fit(X_train, y_train)
        self.models[model_name] = search.best_estimator_
        self.best_params[model_name] = search.best_params_
        
        # Evaluate performance
        y_pred = self.models[model_name].predict(X_test)
        mse = mean_squared_error(y_test, y_pred)
        self.model_performance[model_name].append(mse)
        
        print(f"Best parameters for {model_name}: {search.best_params_}")
        print(f"MSE for {model_name}: {mse}")
    
    def train_lstm(self, X_train, y_train, X_test, y_test):
        model = Sequential([
            LSTM(100, activation='relu', input_shape=(X_train.shape[1], 1), return_sequences=True),
            Dropout(0.2),
            LSTM(50, activation='relu'),
            Dropout(0.2),
            Dense(1)
        ])
        model.compile(optimizer=Adam(learning_rate=0.001), loss='mse')
        
        X_train_reshaped = X_train.reshape((X_train.shape[0], X_train.shape[1], 1))
        X_test_reshaped = X_test.reshape((X_test.shape[0], X_test.shape[1], 1))
        
        early_stopping = EarlyStopping(patience=10, restore_best_weights=True)
        
        tscv = TimeSeriesSplit(n_splits=5)
        for train_index, val_index in tscv.split(X_train_reshaped):
            X_train_split, X_val_split = X_train_reshaped[train_index], X_train_reshaped[val_index]
            y_train_split, y_val_split = y_train.iloc[train_index], y_train.iloc[val_index]
            
            model.fit(
                X_train_split, y_train_split,
                epochs=100,
                batch_size=32,
                validation_data=(X_val_split, y_val_split),
                callbacks=[early_stopping],
                verbose=0
            )
        
        # Evaluate performance
        y_pred = model.predict(X_test_reshaped)
        mse = mean_squared_error(y_test, y_pred)
        self.model_performance['lstm'].append(mse)
        print(f"MSE for LSTM: {mse}")
        
        return model
    
    def train_arima(self, time_series):
        # Use auto_arima for optimal order selection
        model = auto_arima(time_series, start_p=1, start_q=1, max_p=5, max_q=5, m=7,
                           start_P=0, seasonal=True, d=1, D=1, trace=True,
                           error_action='ignore', suppress_warnings=True, stepwise=True)
        
        # Evaluate performance
        y_pred = model.predict(n_periods=len(time_series))
        mse = mean_squared_error(time_series, y_pred)
        self.model_performance['arima'].append(mse)
        print(f"MSE for ARIMA: {mse}")
        
        return model
    
    def update_model_weights(self):
        total_performance = sum(sum(perf) for perf in self.model_performance.values())
        for model in self.models:
            if self.model_performance[model]:
                self.model_weights[model] = 1 - (sum(self.model_performance[model]) / total_performance)
        
        # Normalize weights
        total_weight = sum(self.model_weights.values())
        for model in self.model_weights:
            self.model_weights[model] /= total_weight
    
    def generate_signals(self, latest_data):
        features, _ = self.prepare_features(latest_data)
        scaled_features = self.scaler.transform(features)
        
        signals = {}
        confidences = {}
        
        for model_name in ['random_forest', 'svm', 'xgboost', 'lightgbm', 'gradient_boosting']:
            prediction = self.models[model_name].predict(scaled_features)
            signals[model_name] = 1 if prediction[-1] > latest_data['close'].iloc[-1] else -1
            confidences[model_name] = abs(prediction[-1] - latest_data['close'].iloc[-1]) / latest_data['close'].iloc[-1]
        
        lstm_features = scaled_features.reshape((scaled_features.shape[0], scaled_features.shape[1], 1))
        lstm_prediction = self.models['lstm'].predict(lstm_features)
        signals['lstm'] = 1 if lstm_prediction[-1][0] > latest_data['close'].iloc[-1] else -1
        confidences['lstm'] = abs(lstm_prediction[-1][0] - latest_data['close'].iloc[-1]) / latest_data['close'].iloc[-1]
        
        arima_forecast = self.models['arima'].forecast(steps=1)
        signals['arima'] = 1 if arima_forecast[0] > latest_data['close'].iloc[-1] else -1
        confidences['arima'] = abs(arima_forecast[0] - latest_data['close'].iloc[-1]) / latest_data['close'].iloc[-1]
        
        prophet_forecast = self.models['prophet'].predict(pd.DataFrame({'ds': [latest_data.index[-1] + pd.Timedelta(days=1)]}))
        signals['prophet'] = 1 if prophet_forecast['yhat'].iloc[0] > latest_data['close'].iloc[-1] else -1
        confidences['prophet'] = abs(prophet_forecast['yhat'].iloc[0] - latest_data['close'].iloc[-1]) / latest_data['close'].iloc[-1]
        
        garch_forecast = self.models['garch'].forecast(horizon=1)
        signals['garch'] = 1 if garch_forecast.mean.iloc[-1] > latest_data['close'].iloc[-1] else -1
        confidences['garch'] = abs(garch_forecast.mean.iloc[-1] - latest_data['close'].iloc[-1]) / latest_data['close'].iloc[-1]
        
        return signals, confidences
    
    def ensemble_decision(self, signals, confidences):
        weighted_sum = sum(self.model_weights[model] * signal * confidences[model] for model, signal in signals.items())
        
        if weighted_sum > 0.1:
            return 1  # Strong buy signal
        elif weighted_sum < -0.1:
            return -1  # Strong sell signal
        else:
            return 0  # Hold signal
    
    def calculate_position_size(self, confidence, balance, current_price):
        # Calculate position size based on Kelly Criterion
        win_rate = 0.5 + confidence  # Assume win rate is correlated with confidence
        kelly_fraction = win_rate - ((1 - win_rate) / (self.risk_reward_ratio - 1))
        kelly_fraction = max(0, min(kelly_fraction, 0.2))  # Limit position size to 20% of balance
        return (balance * kelly_fraction) / current_price
    
    def backtest(self, data, initial_balance=10000, risk_reward_ratio=2):
        self.risk_reward_ratio = risk_reward_ratio
        balance = initial_balance
        position = 0
        trades = []
        daily_returns = []
        
        for i in range(len(data) - 30):  # Use 30-day lookback for signal generation
            lookback_data = data.iloc[i:i+30]
            signals, confidences = self.generate_signals(lookback_data)
            decision = self.ensemble_decision(signals, confidences)
            
            price = data['close'].iloc[i+30]
            confidence = abs(sum(self.model_weights[model] * signals[model] * confidences[model] for model in signals))
            
            if decision == 1 and position <= 0:  # Buy signal
                shares_to_buy = self.calculate_position_size(confidence, balance, price)
                position += shares_to_buy
                balance -= shares_to_buy * price * (1 + self.transaction_cost)
                trades.append(('buy', price, shares_to_buy))
            elif decision == -1 and position > 0:  # Sell signal
                balance += position * price * (1 - self.transaction_cost)
                trades.append(('sell', price, position))
                position = 0
            
            # Apply stop-loss and take-profit
            if position > 0:
                if price <= trades[-1][1] * (1 - self.stop_loss):
                    balance += position * price * (1 - self.transaction_cost)
                    trades.append(('stop_loss', price, position))
                    position = 0
                elif price >= trades[-1][1] * (1 + self.take_profit):
                    balance += position * price * (1 - self.transaction_cost)
                    trades.append(('take_profit', price, position))
                    position = 0
            
            daily_return = (balance + position * price) / initial_balance - 1
            daily_returns.append(daily_return)
        
        # Close any remaining position
        if position > 0:
            balance += position * data['close'].iloc[-1] * (1 - self.transaction_cost)
        
        sharpe_ratio = np.sqrt(252) * np.mean(daily_returns) / np.std(daily_returns)
        max_drawdown = np.max(np.maximum.accumulate(daily_returns) - daily_returns)
        
        return balance, trades, sharpe_ratio, max_drawdown
    
    def parallel_backtest(self, data, num_simulations=100):
        results = Parallel(n_jobs=-1)(
            delayed(self.backtest)(data) for _ in range(num_simulations)
        )
        final_balances = [r[0] for r in results]
        sharpe_ratios = [r[2] for r in results]
        max_drawdowns = [r[3] for r in results]
        
        return {
            'mean_balance': np.mean(final_balances),
            'std_balance': np.std(final_balances),
            'mean_sharpe': np.mean(sharpe_ratios),
            'mean_max_drawdown': np.mean(max_drawdowns)
        }
    
    def explain_decision(self, signals, confidences):
        explanation = "Model predictions:\n"
        for model, signal in signals.items():
            explanation += f"{model}: {'Buy' if signal == 1 else 'Sell'} (confidence: {confidences[model]:.2f})\n"
        
        explanation += f"\nModel weights:\n"
        for model, weight in self.model_weights.items():
            explanation += f"{model}: {weight:.2f}\n"
        
        weighted_sum = sum(self.model_weights[model] * signal * confidences[model] for model, signal in signals.items())
        explanation += f"\nWeighted sum: {weighted_sum:.2f}"
        
        if weighted_sum > 0.1:
            explanation += "\nFinal decision: Strong buy"
        elif weighted_sum < -0.1:
            explanation += "\nFinal decision: Strong sell"
        else:
            explanation += "\nFinal decision: Hold"
        
        return explanation
    
    def calculate_shap_values(self, data):
        features, _ = self.prepare_features(data)
        explainer = shap.TreeExplainer(self.models['random_forest'])
        shap_values = explainer.shap_values(features)
        return shap_values, features.columns
    
    def plot_feature_importance(self, data):
        shap_values, feature_names = self.calculate_shap_values(data)
        shap.summary_plot(shap_values, features=data[feature_names], feature_names=feature_names)
    
    def adapt_to_market_regime(self, data):
        # Identify market regime
        returns = data['close'].pct_change()
        volatility = returns.rolling(window=22).std()
        regime = pd.cut(volatility, bins=3, labels=['low', 'medium', 'high'])
        
        # Train separate models for each regime
        for r in ['low', 'medium', 'high']:
            regime_data = data[regime == r]
            self.train_models(regime_data)
        
        self.current_regime = regime.iloc[-1]
    
    def online_update(self, new_data):
        # Periodically retrain models with new data
        self.train_models(new_data)
        
        # Update model weights based on recent performance
        self.update_model_weights()
        
        # Adapt to new market regime
        self.adapt_to_market_regime(new_data)

# Example usage
if __name__ == "__main__":
    # Load your data here
    data = pd.read_csv('your_crypto_data.csv', parse_dates=['timestamp'], index_col='timestamp')
    
    bot = AdvancedCryptoTradingBot()
    
    # Preprocess and train the models
    preprocessed_data = bot.preprocess_data(data)
    bot.train_models(preprocessed_data)
    
    # Generate signals for the latest data point
    latest_data = preprocessed_data.iloc[-30:]  # Use more data points for better signal generation
    signals, confidences = bot.generate_signals(latest_data)
    final_decision = bot.ensemble_decision(signals, confidences)
    
    # print("Individual model signals:", signals)
    print("Final decision:", "Strong Buy" if final_decision == 1 else "Strong Sell" if final_decision == -1 else "Hold")
    
    # Explain the decision
    explanation = bot.explain_decision(signals, confidences)
    print(explanation)
    
    # Perform backtesting
    backtest_results = bot.parallel_backtest(preprocessed_data)
    print("Backtesting results:")
    print(f"Mean final balance: ${backtest_results['mean_balance']:.2f}")
    print(f"Standard deviation of final balance: ${backtest_results['std_balance']:.2f}")
    print(f"Mean Sharpe ratio: {backtest_results['mean_sharpe']:.2f}")
    print(f"Mean maximum drawdown: {backtest_results['mean_max_drawdown']:.2f}")
    
    # Plot feature importance
    bot.plot_feature_importance(preprocessed_data)
    
    # Periodically update the model with new data
    new_data = fetch_new_data()  # Function to fetch new data
    bot.online_update(new_data)

I know it is hard to tell how successful a bot can be, but based on the features included in my bot, what would be your rough guess of my bot's accuracy in percentage