import os
import numpy as np
import pandas as pd
import tensorflow as tf
import matplotlib.pyplot as plt
from sklearn.preprocessing import RobustScaler
from sklearn.model_selection import train_test_split
from tensorflow.keras.models import Sequential, Model, load_model
from tensorflow.keras.layers import (
    Input, 
    LSTM, 
    Dense, 
    Dropout, 
    MultiHeadAttention, 
    Bidirectional, 
    BatchNormalization,
    GlobalAveragePooling1D,
    Layer
)
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.regularizers import l2
import logging
import seaborn as sns
from sklearn.metrics import confusion_matrix, mean_absolute_percentage_error, r2_score


# Configuration globale
CONFIG = {
    'DATA_DIR': 'data',
    'MODEL_DIR': 'models',
    'LOG_DIR': 'logs',
    'REQUIRED_COLUMNS': ['date', 'open', 'high', 'low', 'close', 'volume'],
    'DEFAULT_TIMEFRAMES': ['30M', '1H', '4H', 'D']
}

# Configuration du logging
os.makedirs(CONFIG['LOG_DIR'], exist_ok=True)
logging.basicConfig(
    filename=os.path.join(CONFIG['LOG_DIR'], 'trading_predictor.log'),
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s'
)



class IchimokuCalculator:
    @staticmethod
    def calculate_ichimoku(df, conversion_line_period=9, base_line_period=26, 
                          leading_span_b_period=52, displacement=26):
        """
        Calcule tous les composants de l'indicateur Ichimoku Cloud
        
        Args:
            df (pandas.DataFrame): DataFrame avec colonnes high, low, close
            conversion_line_period (int): Période pour Tenkan-sen
            base_line_period (int): Période pour Kijun-sen
            leading_span_b_period (int): Période pour Senkou Span B
            displacement (int): Période de décalage
            
        Returns:
            pandas.DataFrame: DataFrame avec les indicateurs Ichimoku ajoutés
        """
        df = df.copy()
        
        # Tenkan-sen (Conversion Line)
        high_values = df['high'].rolling(window=conversion_line_period).max()
        low_values = df['low'].rolling(window=conversion_line_period).min()
        df['tenkan_sen'] = (high_values + low_values) / 2

        # Kijun-sen (Base Line)
        high_values = df['high'].rolling(window=base_line_period).max()
        low_values = df['low'].rolling(window=base_line_period).min()
        df['kijun_sen'] = (high_values + low_values) / 2

        # Senkou Span A (Leading Span A)
        df['senkou_span_a'] = ((df['tenkan_sen'] + df['kijun_sen']) / 2).shift(displacement)

        # Senkou Span B (Leading Span B)
        high_values = df['high'].rolling(window=leading_span_b_period).max()
        low_values = df['low'].rolling(window=leading_span_b_period).min()
        df['senkou_span_b'] = ((high_values + low_values) / 2).shift(displacement)

        # Chikou Span (Lagging Span)
        df['chikou_span'] = df['close'].shift(-displacement)

        # Calcul des signaux basés sur l'Ichimoku
        df['ichimoku_signal'] = 0
        
        # Signal d'achat
        buy_signal = (
            (df['tenkan_sen'] > df['kijun_sen']) & 
            (df['close'] > df['senkou_span_a']) & 
            (df['close'] > df['senkou_span_b']) &
            (df['chikou_span'].shift(displacement) > df['close'].shift(displacement))
        )
        df.loc[buy_signal, 'ichimoku_signal'] = 1

        # Signal de vente
        sell_signal = (
            (df['tenkan_sen'] < df['kijun_sen']) & 
            (df['close'] < df['senkou_span_a']) & 
            (df['close'] < df['senkou_span_b']) &
            (df['chikou_span'].shift(displacement) < df['close'].shift(displacement))
        )
        df.loc[sell_signal, 'ichimoku_signal'] = -1

        # Caractéristiques supplémentaires
        df['tenkan_kijun_cross'] = np.where(df['tenkan_sen'] > df['kijun_sen'], 1, -1)
        df['price_relative_to_kumo'] = np.where(
            (df['close'] > df['senkou_span_a']) & (df['close'] > df['senkou_span_b']), 1,
            np.where((df['close'] < df['senkou_span_a']) & (df['close'] < df['senkou_span_b']), -1, 0)
        )
        
        return df

class AttentionLSTM:
    def __init__(self, input_shape, lstm_units=128):
        self.input_shape = input_shape
        self.lstm_units = lstm_units
        self.model = self.build_model()
        self.compile_model()

    def build_lstm_block(self, inputs):
        x = BatchNormalization()(inputs)
        lstm_out = Bidirectional(LSTM(
            self.lstm_units,
            return_sequences=True,
            dropout=0.2,
            recurrent_dropout=0.2,
            kernel_regularizer=l2(1e-5)
        ))(x)
        return lstm_out

    def build_attention_block(self, lstm_out):
        attention_output = MultiHeadAttention(
            num_heads=4,
            key_dim=self.lstm_units,
            dropout=0.1
        )(lstm_out, lstm_out)
        
        attention_output = BatchNormalization()(attention_output)
        pooled = GlobalAveragePooling1D()(attention_output)
        return pooled

    def build_model(self):
        inputs = Input(shape=self.input_shape)
        
        lstm_out = self.build_lstm_block(inputs)
        context = self.build_attention_block(lstm_out)
        
        x = Dense(64, activation='relu')(context)
        x = BatchNormalization()(x)
        x = Dropout(0.3)(x)
        
        x = Dense(32, activation='relu')(x)
        x = BatchNormalization()(x)
        x = Dropout(0.2)(x)
        
        output = Dense(1, activation='linear')(x)
        
        return Model(inputs, output)
    
    def compile_model(self):
        self.model.compile(
            optimizer=Adam(learning_rate=0.001),
            loss='huber',
            metrics=['mae', 'mse']
        )


class TradingPredictor:
    def __init__(self, seq_length=120, batch_size=32, epochs=50, validation_split=0.2):
        self.seq_length = seq_length
        self.batch_size = batch_size
        self.epochs = epochs
        self.validation_split = validation_split
        
        self.feature_scaler = RobustScaler()
        self.target_scaler = RobustScaler()

        self.model_dir = 'models'
        os.makedirs(self.model_dir, exist_ok=True)

        self.n_features = None  # Augmenté pour inclure les features Ichimoku
        self.models = None
        self.init_models()
        self.logger = logging.getLogger(__name__)

    def init_models(self):
        for idx, model in enumerate(self.models):
            try:
                model_path = os.path.join(self.model_dir, f'attention_model_{idx}.keras')
                model.model = load_model(model_path)
                print(f"Modèle {idx} chargé depuis {model_path}.")
            except:
                print(f"Modèle {idx} non trouvé, entraînement nécessaire.")
                
    def validate_data(self, df):
        """Validation des données d'entrée"""
        missing_cols = [col for col in CONFIG['REQUIRED_COLUMNS'] if col not in df.columns]
        if missing_cols:
            self.logger.error(f"Colonnes manquantes: {missing_cols}")
            raise ValueError(f"Colonnes manquantes: {missing_cols}")
        
        null_counts = df[CONFIG['REQUIRED_COLUMNS']].isnull().sum()
        if null_counts.any():
            self.logger.warning(f"Valeurs nulles détectées: {null_counts[null_counts > 0]}")
    
    def handle_outliers(self, df, columns=['high', 'low', 'close'], n_std=3):
        """Traitement des valeurs aberrantes"""
        df_clean = df.copy()
        for col in columns:
            mean = df_clean[col].mean()
            std = df_clean[col].std()
            df_clean[col] = df_clean[col].clip(mean - n_std * std, mean + n_std * std)
        return df_clean
    
    def calculate_metrics(self, y_true, y_pred):
        """Calcul des métriques de performance"""
        metrics = {
            'mape': mean_absolute_percentage_error(y_true, y_pred),
            'r2': r2_score(y_true, y_pred),
            'hit_rate': np.mean(np.sign(np.diff(y_true)) == np.sign(np.diff(y_pred)))
        }
        self.logger.info(f"Métriques de performance: {metrics}")
        return metrics
    
    def run_backtesting(self, test_data, initial_balance=10000):
        """Simulation de trading"""
        balance = initial_balance
        position = None
        trades = []
        
        for i in range(len(test_data) - 1):
            current_price = test_data.iloc[i]['close']
            next_price = test_data.iloc[i + 1]['close']
            prediction = self.predict(test_data.iloc[i:i+1])
            
            if prediction > current_price and position is None:
                position = current_price
                trades.append(('BUY', current_price))
                self.logger.info(f"BUY signal at {current_price}")
            elif prediction < current_price and position is not None:
                profit = current_price - position
                balance += profit
                position = None
                trades.append(('SELL', current_price))
                self.logger.info(f"SELL signal at {current_price}, profit: {profit}")
        
        final_performance = {
            'initial_balance': initial_balance,
            'final_balance': balance,
            'return': ((balance - initial_balance) / initial_balance) * 100,
            'n_trades': len(trades)
        }
        self.logger.info(f"Backtest results: {final_performance}")
        return final_performance, trades
    

    def add_technical_indicators(self, df):
        """
        Ajoute tous les indicateurs techniques au DataFrame
        """
        df_copy = df.copy()
        
        # Ajout des indicateurs Ichimoku
        df_copy = IchimokuCalculator.calculate_ichimoku(df_copy)
        
        # Moyennes mobiles
        for period in [5, 8, 13, 21, 34, 55, 89]:
            df_copy[f'ema_{period}'] = df_copy['close'].ewm(span=period, adjust=False).mean()
            df_copy[f'sma_{period}'] = df_copy['close'].rolling(window=period).mean()
        
        # RSI
        for period in [7, 14, 21]:
            delta = df_copy['close'].diff()
            gain = (delta.where(delta > 0, 0)).rolling(window=period).mean()
            loss = (-delta.where(delta < 0, 0)).rolling(window=period).mean()
            rs = gain / loss
            df_copy[f'rsi_{period}'] = 100 - (100 / (1 + rs))

        # Volatilité et Momentum
        df_copy['volatility'] = df_copy['close'].rolling(window=20).std()
        df_copy['momentum'] = df_copy['close'].diff(periods=10)
        
        # Caractéristiques dérivées de l'Ichimoku
        df_copy['tenkan_kijun_diff'] = df_copy['tenkan_sen'] - df_copy['kijun_sen']
        df_copy['price_to_kumo_top'] = df_copy['close'] - df_copy['senkou_span_a']
        df_copy['price_to_kumo_bottom'] = df_copy['close'] - df_copy['senkou_span_b']
        df_copy['kumo_thickness'] = df_copy['senkou_span_a'] - df_copy['senkou_span_b']
        
        # Sélection des colonnes numériques uniquement
        numeric_columns = df_copy.select_dtypes(include=[np.number]).columns
        df_copy = df_copy[numeric_columns]
        
        return df_copy.ffill().bfill()


    def prepare_data(self, df):
        """
        Prépare les données pour l'entraînement ou la prédiction
        """
        df = self.add_technical_indicators(df)
        
        features = [col for col in df.columns if col not in ['date', 'close', 'timeframe'] 
                   and df[col].dtype in ['float64', 'int64']]
        
        print("Features utilisées:", features)
        
        # Mise à jour du nombre de features
        self.n_features = len(features)
        print("Nombre de features:", self.n_features)
        
        # Initialisation des modèles si ce n'est pas déjà fait
        if self.models is None:
            self.models = [AttentionLSTM(input_shape=(self.seq_length, self.n_features)) 
                         for _ in range(2)]
            self.init_models()

        x_data = df[features].values
        y_data = df['close'].values.reshape(-1, 1)

        x_data = self.feature_scaler.fit_transform(x_data)
        y_data = self.target_scaler.fit_transform(y_data)

        x, y = [], []
        for i in range(len(df) - self.seq_length):
            x.append(x_data[i:i+self.seq_length])
            y.append(y_data[i+self.seq_length])
        
        return np.array(x), np.array(y)


    def train(self, df, update_only=False):
        """
        Entraîne ou met à jour les modèles
        """
        x, y = self.prepare_data(df)
        x_train, x_val, y_train, y_val = train_test_split(x, y, test_size=self.validation_split, shuffle=False)

        callbacks = [
            EarlyStopping(monitor='val_loss', patience=15, restore_best_weights=True),
            ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5, min_lr=1e-6),
            ModelCheckpoint(
                os.path.join(self.model_dir, 'best_model.keras'),
                monitor='val_loss',
                save_best_only=True
            )
        ]

        for idx, model in enumerate(self.models):
            if update_only:
                print(f"\nMise à jour du modèle {idx}...")
                history = model.model.fit(
                    x_train, y_train,
                    epochs=10,
                    batch_size=self.batch_size,
                    shuffle=False,
                    verbose=1
                )
            else:
                print(f"\nEntraînement complet du modèle {idx}...")
                history = model.model.fit(
                    x_train, y_train,
                    validation_data=(x_val, y_val),
                    epochs=self.epochs,
                    batch_size=self.batch_size,
                    callbacks=callbacks,
                    verbose=1
                )
            
            model_path = os.path.join(self.model_dir, f'attention_model_{idx}.keras')
            model.model.save(model_path)
            print(f"Modèle {idx} sauvegardé dans {model_path}")


    def predict(self, df_recent):
        """
        Fait une prédiction pour la prochaine période
        """
        x, _ = self.prepare_data(df_recent)
        if len(x) < 1:
            raise ValueError("Pas assez de données pour faire une prédiction")
            
        predictions = []
        for model in self.models:
            pred = model.model.predict(x[-1:], verbose=0)
            predictions.append(pred)
            
        avg_pred = np.mean(predictions, axis=0)
        return self.target_scaler.inverse_transform(avg_pred)[0][0]
    
    
    def plot_predictions(self, timeframe='1H', n_points=100):
        """
        Affiche un graphique avec les prédictions et l'Ichimoku
        """
        try:
            df_recent = load_data([timeframe])
            df_recent = IchimokuCalculator.calculate_ichimoku(df_recent)
            
            x, y_real = self.prepare_data(df_recent)
            
            x = x[-n_points:]
            y_real = y_real[-n_points:]
            
            predictions = []
            for model in self.models:
                pred = model.model.predict(x, verbose=0)
                predictions.append(pred)
            
            y_pred = np.mean(predictions, axis=0)
            
            y_real = self.target_scaler.inverse_transform(y_real)
            y_pred = self.target_scaler.inverse_transform(y_pred)
            
            dates = df_recent['date'].iloc[-n_points:].values
            
            # Création du graphique
            fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(15, 12), gridspec_kw={'height_ratios': [3, 1]})
            
            # Graphique principal
            ax1.plot(dates, y_real, label='Prix réel', color='blue', linewidth=1)
            ax1.plot(dates, y_pred, label='Prédictions', color='red', linestyle='--', linewidth=1)
            
            # Ichimoku
            recent_data = df_recent.iloc[-n_points:]
            ax1.plot(dates, recent_data['tenkan_sen'], label='Tenkan-sen', color='gray', linewidth=0.5)
            ax1.plot(dates, recent_data['kijun_sen'], label='Kijun-sen', color='brown', linewidth=0.5)
            ax1.fill_between(dates, 
                           recent_data['senkou_span_a'], 
                           recent_data['senkou_span_b'], 
                           where=recent_data['senkou_span_a'] >= recent_data['senkou_span_b'],
                           color='green', 
                           alpha=0.1, 
                           label='Kumo (bull)')
            
            ax1.fill_between(dates, 
                           recent_data['senkou_span_a'], 
                           recent_data['senkou_span_b'], 
                           where=recent_data['senkou_span_a'] < recent_data['senkou_span_b'],
                           color='red', 
                           alpha=0.1, 
                           label='Kumo (bear)')

            # Configuration du graphique principal
            ax1.set_title(f'Prédictions et Ichimoku Cloud ({timeframe})')
            ax1.set_xlabel('Date')
            ax1.set_ylabel('Prix')
            ax1.legend(loc='upper left')
            ax1.grid(True, alpha=0.3)
            
            # Calcul et tracé de l'erreur
            error = np.abs(y_real - y_pred)
            ax2.plot(dates, error, color='purple', label='Erreur absolue')
            ax2.fill_between(dates, 0, error, alpha=0.2, color='purple')
            
            # Configuration du graphique d'erreur
            ax2.set_title('Erreur de prédiction')
            ax2.set_xlabel('Date')
            ax2.set_ylabel('Erreur absolue')
            ax2.legend()
            ax2.grid(True, alpha=0.3)
            
            # Rotation des dates pour une meilleure lisibilité
            for ax in [ax1, ax2]:
                ax.tick_params(axis='x', rotation=45)
            
            plt.tight_layout()
            plt.show()
            
            # Calcul et affichage des métriques d'erreur
            mae = np.mean(np.abs(y_real - y_pred))
            mse = np.mean((y_real - y_pred) ** 2)
            rmse = np.sqrt(mse)
            
            print("\nMétriques d'erreur:")
            print(f"MAE: {mae:.2f}")
            print(f"MSE: {mse:.2f}")
            print(f"RMSE: {rmse:.2f}")
            
        except Exception as e:
            print(f"Erreur lors de la création du graphique: {str(e)}")
            
    def plot_feature_importance(self, features):
        """Visualisation de l'importance des features"""
        plt.figure(figsize=(12, 6))
        feature_importance = pd.DataFrame({
            'feature': features,
            'importance': range(len(features))  # À remplacer par les vraies importances si disponibles
        }).sort_values('importance', ascending=False)
        
        sns.barplot(data=feature_importance, x='importance', y='feature')
        plt.title('Feature Importance')
        plt.tight_layout()
        plt.show()

    def plot_confusion_matrix(self, y_true, y_pred):
        """Visualisation de la matrice de confusion"""
        y_true_dir = np.sign(np.diff(y_true.flatten()))
        y_pred_dir = np.sign(np.diff(y_pred.flatten()))
        
        cm = confusion_matrix(y_true_dir, y_pred_dir)
        plt.figure(figsize=(8, 6))
        sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')
        plt.title('Confusion Matrix - Direction Predictions')
        plt.xlabel('Predicted')
        plt.ylabel('Actual')
        plt.show()

def load_data(timeframes):
    all_data = []
    
    for timeframe in timeframes:
        filename = f"data_volatility_{timeframe}.csv"
        if os.path.exists(filename):
            try:
                df = pd.read_csv(
                    filename,
                    encoding='utf-16',
                    names=["date", "open", "high", "low", "close", "volume", "adj_close"]
                )
                
                if timeframe == 'D':
                    df['date'] = pd.to_datetime(df['date'].str.strip(), format='%Y.%m.%d')
                else:
                    df['date'] = pd.to_datetime(df['date'].str.strip(), format='%Y.%m.%d %H:%M')
                    
                df['timeframe'] = timeframe
                all_data.append(df)
                print(f"Fichier {filename} chargé avec succès.")
            except Exception as e:
                print(f"Erreur lors du chargement de {filename}: {str(e)}")
    
    if all_data:
        return pd.concat(all_data).drop_duplicates().sort_values("date").reset_index(drop=True)
    else:
        raise FileNotFoundError("Aucun fichier de données valide trouvé.")

def main():
    predictor = TradingPredictor()

    while True:
        print("\n=== Système de Prédiction ===")
        print("1: Entraîner le modèle avec de nouvelles données")
        print("2: Mettre à jour le modèle avec des données incrémentales")
        print("3: Prédire le prochain prix")
        print("4: Afficher le graphique des prédictions")
        print("5: Exécuter le backtesting")
        print("6: Afficher les métriques de performance")
        print("7: Visualiser l'importance des features")
        print("8: Quitter")

        choice = input("\nChoisissez une option (1-8) : ")
        
        if choice == '1':
            print("\nEntraînement complet du modèle...")
            try:
                timeframes = ['30M', '1H', '4H', 'D']
                df_all = load_data(timeframes)
                predictor.train(df_all, update_only=False)
                print("\nModèle entièrement entraîné et enregistré.")
            except Exception as e:
                print(f"\nErreur lors de l'entraînement : {str(e)}")
        
        elif choice == '2':
            print("\nMise à jour du modèle avec des données récentes...")
            try:
                timeframes = ['1H']
                df_all = load_data(timeframes)
                predictor.train(df_all, update_only=True)
                print("\nModèle mis à jour avec succès.")
            except Exception as e:
                print(f"\nErreur lors de la mise à jour : {str(e)}")
        
        elif choice == '3':
            print("\nPrédiction du prochain prix...")
            try:
                timeframes = ['1H']
                df_recent = load_data(timeframes).tail(predictor.seq_length + 1)
                prediction = predictor.predict(df_recent)
                print(f"\nPrédiction du prochain prix : {prediction:.2f}")
            except Exception as e:
                print(f"\nErreur lors de la prédiction : {str(e)}")
        
        elif choice == '4':
            try:
                predictor.plot_predictions()
            except Exception as e:
                print(f"\nErreur lors de l'affichage du graphique : {str(e)}")

       
        if choice == '5':
            try:
                timeframes = ['1H']
                df_test = load_data(timeframes)
                results, trades = predictor.run_backtesting(df_test)
                print("\nRésultats du backtesting:")
                print(f"Balance finale: {results['final_balance']:.2f}")
                print(f"Rendement: {results['return']:.2f}%")
                print(f"Nombre de trades: {results['n_trades']}")
            except Exception as e:
                print(f"\nErreur lors du backtesting : {str(e)}")
        
        elif choice == '6':
            try:
                timeframes = ['1H']
                df_test = load_data(timeframes)
                x_test, y_test = predictor.prepare_data(df_test)
                y_pred = predictor.predict(df_test)
                metrics = predictor.calculate_metrics(y_test, y_pred)
                print("\nMétriques de performance:")
                for metric, value in metrics.items():
                    print(f"{metric}: {value:.4f}")
            except Exception as e:
                print(f"\nErreur lors du calcul des métriques : {str(e)}")
        
        elif choice == '7':
            try:
                predictor.plot_feature_importance(predictor.features)
            except Exception as e:
                print(f"\nErreur lors de la visualisation : {str(e)}")
        
        elif choice == '8':
            print("\nFermeture du programme.")
            break

if __name__ == "__main__":
    main();;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;quand jexecute jai cette erreur: PS C:\Users\fchabi\Downloads\projet> python projetv4.py
2024-11-08 22:13:18.779132: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-11-08 22:13:20.219604: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
Traceback (most recent call last):
  File "C:\Users\fchabi\Downloads\projet\projetv4.py", line 630, in <module>
    main()
  File "C:\Users\fchabi\Downloads\projet\projetv4.py", line 542, in main
    predictor = TradingPredictor()
  File "C:\Users\fchabi\Downloads\projet\projetv4.py", line 187, in __init__
    self.init_models()
  File "C:\Users\fchabi\Downloads\projet\projetv4.py", line 191, in init_models
    for idx, model in enumerate(self.models):
TypeError: 'NoneType' object is not iterable
PS C:\Users\fchabi\Downloads\projet>