This script needs to take one big trade between 9AM and 11AM, only one big trade on each pair : # Install required libraries (if not already installed)
# Uncomment the lines below if you need to install the libraries

# !pip install yfinance backtrader pandas_ta ta mplfinance matplotlib tensorflow scikit-learn

# Import necessary libraries
import pandas as pd
import numpy as np
import datetime
import pandas_ta as ta
import matplotlib.pyplot as plt
import mplfinance as mpf
import yfinance as yf
import warnings
import pytz
import os

# Optional: Set environment variable to disable oneDNN optimizations if desired
# os.environ['TF_ENABLE_ONEDNN_OPTS'] = '0'

# For Backtesting
import backtrader as bt

# For AI Model
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Dense, Dropout
from tensorflow.keras.callbacks import EarlyStopping

# Suppress warnings
warnings.filterwarnings('ignore')

# List of currency pairs
currency_pairs = ['EURUSD=X', 'GBPUSD=X', 'USDJPY=X']  # yfinance symbols for Forex pairs

# Define features and sequence length before usage
features = ['Close', 'RSI', 'ATR', 'Institutional_Flow', 'BOS_Up', 'BOS_Down',
            'CHoCH_Up', 'CHoCH_Down', 'Equal_Highs', 'Equal_Lows', 'Hour']
sequence_length = 60  # Number of time steps for LSTM

# Function to fetch data using yfinance
def fetch_data(pair, interval='1m'):
    try:
        # Fetch data
        data = yf.download(tickers=pair, interval=interval, period='5d', progress=False)
        if data.empty:
            print(f"No data fetched for {pair}")
            return None

        # Ensure datetime index is timezone-aware in UTC
        if data.index.tz is None:
            data.index = data.index.tz_localize('UTC')  # Set timezone to UTC
        else:
            data.index = data.index.tz_convert('UTC')  # Convert to UTC

        data.rename(columns={
            'Open': 'Open',
            'High': 'High',
            'Low': 'Low',
            'Close': 'Close',
            'Volume': 'Volume'
        }, inplace=True)
        return data
    except Exception as e:
        print(f"Error fetching data for {pair}: {e}")
        return None

# Collect Data
market_data = {}
for pair in currency_pairs:
    print(f"Fetching data for {pair}...")
    data = fetch_data(pair, interval='1m')
    if data is not None:
        key = pair.replace('=X', '').replace('/', '_')
        market_data[key] = data

# Implement ICT Utilities and AI Model Integration

# 1. Feature Engineering and Indicator Calculation
def prepare_data(df):
    df = df.copy()
    # Calculate technical indicators
    df['RSI'] = ta.rsi(df['Close'], length=14)
    df['ATR'] = ta.atr(df['High'], df['Low'], df['Close'], length=14)
    df['EMA_50'] = ta.ema(df['Close'], length=50)
    df['EMA_200'] = ta.ema(df['Close'], length=200)
    df['Hour'] = df.index.hour  # Time feature for Kill Zones

    # Calculate market structure
    df = detect_market_structure(df)
    df = identify_liquidity_pools(df)
    df = institutional_order_flow(df)

    # Convert boolean columns to integers
    boolean_columns = ['BOS_Up', 'BOS_Down', 'CHoCH_Up', 'CHoCH_Down', 'Equal_Highs', 'Equal_Lows']
    df[boolean_columns] = df[boolean_columns].astype(int)

    # Drop rows with NaN values
    df.dropna(inplace=True)

    return df

# Advanced Market Structure Detection
def detect_market_structure(df):
    df['High_Shift1'] = df['High'].shift(1)
    df['Low_Shift1'] = df['Low'].shift(1)
    df['High_Shift2'] = df['High'].shift(2)
    df['Low_Shift2'] = df['Low'].shift(2)

    # Swing Highs and Lows
    df['Swing_High'] = (df['High'] > df['High_Shift1']) & (df['High_Shift1'] > df['High_Shift2'])
    df['Swing_Low'] = (df['Low'] < df['Low_Shift1']) & (df['Low_Shift1'] < df['Low_Shift2'])

    # Break of Structure (BOS)
    df['BOS_Up'] = ((df['Close'] > df['High_Shift1']) & df['Swing_High'].shift(1))
    df['BOS_Down'] = ((df['Close'] < df['Low_Shift1']) & df['Swing_Low'].shift(1))

    # Change of Character (CHoCH)
    df['CHoCH_Up'] = df['BOS_Up'] & (df['Close'] > df['High_Shift2'])
    df['CHoCH_Down'] = df['BOS_Down'] & (df['Close'] < df['Low_Shift2'])

    return df

# Liquidity Pool Identification
def identify_liquidity_pools(df):
    df['Equal_Highs'] = (np.isclose(df['High'], df['High_Shift1'], atol=1e-5) &
                         np.isclose(df['High'], df['High_Shift2'], atol=1e-5))
    df['Equal_Lows'] = (np.isclose(df['Low'], df['Low_Shift1'], atol=1e-5) &
                        np.isclose(df['Low'], df['Low_Shift2'], atol=1e-5))
    return df

# Institutional Order Flow Analysis (Approximation)
def institutional_order_flow(df):
    df['Institutional_Flow'] = df['EMA_50'] - df['EMA_200']
    return df

# Prepare datasets
ai_models = {}
for key in market_data.keys():
    df = market_data[key]
    df_prepared = prepare_data(df)
    market_data[key] = df_prepared

    # Define target variable: Next period return (classification problem)
    df_prepared['Target'] = (df_prepared['Close'].shift(-1) > df_prepared['Close']).astype(int)
    df_prepared.dropna(inplace=True)

    if len(df_prepared) < sequence_length:
        print(f"Not enough data for {key} to create sequences.")
        continue

    # Features for AI model
    X = df_prepared[features]
    y = df_prepared['Target']

    # Scaling
    scaler = MinMaxScaler()
    X_scaled = scaler.fit_transform(X)

    # Create sequences
    X_seq, y_seq = [], []
    for i in range(sequence_length, len(X_scaled)):
        X_seq.append(X_scaled[i-sequence_length:i])
        y_seq.append(y.iloc[i])
    X_seq, y_seq = np.array(X_seq), np.array(y_seq)

    if len(X_seq) == 0:
        print(f"Not enough data for {key} after sequence creation.")
        continue

    # Split data
    X_train, X_test, y_train, y_test = train_test_split(X_seq, y_seq, test_size=0.2, shuffle=False)

    # Build LSTM model
    model = Sequential()
    model.add(LSTM(64, return_sequences=True, input_shape=(X_train.shape[1], X_train.shape[2])))
    model.add(Dropout(0.2))
    model.add(LSTM(32))
    model.add(Dropout(0.1))
    model.add(Dense(1, activation='sigmoid'))
    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])

    # Early stopping
    early_stopping = EarlyStopping(monitor='val_loss', patience=3)

    # Train model
    history = model.fit(
        X_train, y_train,
        epochs=5,  # Reduced epochs for quick testing
        batch_size=32,
        validation_data=(X_test, y_test),
        callbacks=[early_stopping],
        verbose=1
    )

    # Evaluate model
    loss, accuracy = model.evaluate(X_test, y_test)
    print(f'{key} Model Accuracy: {accuracy*100:.2f}%')

    ai_models[key] = (model, scaler)

# Adjusted Kill Zone Definition
def is_within_kill_zone(dt):
    # Kill Zone from 9 AM to 11 AM Paris Time
    paris_timezone = pytz.timezone('Europe/Paris')
    dt_paris = dt.astimezone(paris_timezone)
    return dt_paris.time() >= datetime.time(9, 0) and dt_paris.time() < datetime.time(11, 0)

# Define the Enhanced ICT Strategy with AI Integration
class EnhancedICTStrategyWithAI(bt.Strategy):
    params = dict(
        verbose=True,  # Enable verbose for debugging
        max_risk_per_trade=0.01,  # 1% risk per trade
        sequence_length=sequence_length,
        features=features,
        model=None,
        scaler=None
    )

    def __init__(self):
        self.trade_journal = []
        self.order = None

        self.sequence_length = self.params.sequence_length
        self.features = self.params.features

        # Access data
        self.data_open = self.datas[0].open
        self.data_high = self.datas[0].high
        self.data_low = self.datas[0].low
        self.data_close = self.datas[0].close

        # Indicators
        self.atr = self.datas[0].ATR
        self.inst_flow = self.datas[0].Institutional_Flow
        self.rsi = self.datas[0].RSI

        # AI Model
        self.model = self.params.model
        self.scaler = self.params.scaler
        self.past_data = []

    def next(self):
        dt = self.datas[0].datetime.datetime(0)

        # Check if within Kill Zone
        if not is_within_kill_zone(dt):
            return

        # Collect features for AI prediction
        current_features = [
            self.data_close[0],
            self.rsi[0],
            self.atr[0],
            self.inst_flow[0],
            int(self.datas[0].BOS_Up[0]),
            int(self.datas[0].BOS_Down[0]),
            int(self.datas[0].CHoCH_Up[0]),
            int(self.datas[0].CHoCH_Down[0]),
            int(self.datas[0].Equal_Highs[0]),
            int(self.datas[0].Equal_Lows[0]),
            dt.hour
        ]
        self.past_data.append(current_features)
        if len(self.past_data) < self.sequence_length:
            return
        elif len(self.past_data) > self.sequence_length:
            self.past_data.pop(0)

        # Prepare AI input
        X_ai = np.array(self.past_data)
        X_ai_scaled = self.scaler.transform(X_ai)
        X_ai_scaled = X_ai_scaled.reshape(1, self.sequence_length, len(self.features))
        ai_prediction = self.model.predict(X_ai_scaled)[0][0]

        # Adjust AI signal threshold to be less strict
        ai_signal = ai_prediction > 0.4  # Reduced threshold to 0.4

        # Simplify Entry Conditions
        if not self.position:
            if is_within_kill_zone(dt):
                self.enter_long_position()
        elif self.position.size > 0:
            # Close position at 11 AM Paris Time
            paris_timezone = pytz.timezone('Europe/Paris')
            dt_paris = dt.astimezone(paris_timezone)
            if dt_paris.time() >= datetime.time(11, 0):
                self.close_position()

    def enter_long_position(self):
        atr = self.atr[-1]
        stop_loss_price = self.data_close[0] - 1 * atr
        take_profit_price = self.data_close[0] + 1 * atr  # Adjusted to 1x ATR for quicker TP

        stop_loss_pips = (self.data_close[0] - stop_loss_price) * 10000  # Convert to pips
        position_size = self.calculate_position_size(stop_loss_pips)
        position_size = max(position_size, 0.01)  # Minimum position size
        position_size = min(position_size, 10)    # Maximum position size

        self.order = self.buy(size=position_size)
        self.log_trade('BUY', position_size)
        # Set Stop Loss and Take Profit
        self.sell(exectype=bt.Order.Stop, price=stop_loss_price, size=position_size)
        self.sell(exectype=bt.Order.Limit, price=take_profit_price, size=position_size)

        print(f"Entering long position at {self.data_close[0]} on {self.datas[0].datetime.datetime(0)}")

    def close_position(self):
        self.order = self.close()
        position_type = 'LONG' if self.position.size > 0 else 'SHORT'
        self.log_trade(f'CLOSE {position_type}', abs(self.position.size))

        print(f"Closing position at {self.data_close[0]} on {self.datas[0].datetime.datetime(0)}")

    def calculate_position_size(self, stop_loss_pips):
        # Position size in lots
        risk_amount = self.broker.getvalue() * self.params.max_risk_per_trade
        pip_value = 10  # For standard lots in major currency pairs
        position_size = risk_amount / (stop_loss_pips * pip_value)
        return position_size

    def log_trade(self, action, position_size):
        dt = self.datas[0].datetime.datetime(0)
        price = self.data_close[0]
        log_entry = {
            'Date': dt,
            'Action': action,
            'Price': price,
            'Position Size': position_size
        }
        self.trade_journal.append(log_entry)
        if self.params.verbose:
            print(f"{dt} - {action} at {price}, Position Size: {position_size}")
            print(f"Trade logged: {log_entry}")

# Custom Data Feed to Include Custom Columns
class CustomPandasData(bt.feeds.PandasData):
    lines = ('ATR', 'Institutional_Flow', 'RSI', 'BOS_Up', 'BOS_Down',
             'CHoCH_Up', 'CHoCH_Down', 'Equal_Highs', 'Equal_Lows')
    params = (
        ('datetime', None),
        ('open', 'Open'),
        ('high', 'High'),
        ('low', 'Low'),
        ('close', 'Close'),
        ('volume', 'Volume'),
        ('openinterest', -1),
        ('ATR', 'ATR'),
        ('Institutional_Flow', 'Institutional_Flow'),
        ('RSI', 'RSI'),
        ('BOS_Up', 'BOS_Up'),
        ('BOS_Down', 'BOS_Down'),
        ('CHoCH_Up', 'CHoCH_Up'),
        ('CHoCH_Down', 'CHoCH_Down'),
        ('Equal_Highs', 'Equal_Highs'),
        ('Equal_Lows', 'Equal_Lows'),
    )

# Run Backtest
results_dict = {}
for key in market_data.keys():
    print(f"Running backtest for {key}...")
    df = market_data[key].copy()
    df.dropna(inplace=True)

    if key not in ai_models:
        print(f"Skipping {key} due to insufficient data for AI model.")
        continue

    # Get AI model and scaler for this currency pair
    model, scaler = ai_models[key]

    data_feed = CustomPandasData(dataname=df)

    cerebro = bt.Cerebro()
    cerebro.adddata(data_feed)
    cerebro.addstrategy(
        EnhancedICTStrategyWithAI,
        verbose=True,  # Enable verbose mode for diagnostics
        model=model,
        scaler=scaler,
        sequence_length=sequence_length,
        features=features
    )
    cerebro.broker.setcash(100000.0)
    cerebro.broker.setcommission(commission=0.0002)

    # Run backtest
    result = cerebro.run()
    final_value = cerebro.broker.getvalue()
    print(f"Final Portfolio Value for {key}: {final_value:.2f}")

    # Store results and trade journal
    trade_journal = result[0].trade_journal
    trade_journal_df = pd.DataFrame(trade_journal)
    results_dict[key] = {
        'Final Portfolio Value': final_value,
        'Trade Journal': trade_journal_df,
        'Cerebro': cerebro
    }

# Plot Trades on Price Chart
for key, result in results_dict.items():
    df = market_data[key].copy()
    trade_journal = result['Trade Journal']
    df = df.tail(500)

    if trade_journal.empty:
        print(f"No trades recorded for {key}.")
        continue

    if 'Action' not in trade_journal.columns:
        print(f"'Action' column not found in trade journal for {key}.")
        print(trade_journal)
        continue

    # Create buy and sell signals
    buy_signals = trade_journal[trade_journal['Action'] == 'BUY']
    sell_signals = trade_journal[trade_journal['Action'] == 'SELL']
    close_signals = trade_journal[trade_journal['Action'].str.startswith('CLOSE')]

    # Prepare addplots
    apds = []

    # Initialize signal columns to NaN
    df['Buy_Signal'] = np.nan
    df['Sell_Signal'] = np.nan
    df['Close_Signal'] = np.nan

    if not buy_signals.empty:
        buy_dates = pd.to_datetime(buy_signals['Date'])
        df.loc[df.index.isin(buy_dates), 'Buy_Signal'] = df['Low'] - (df['High'] - df['Low'])*0.02
        if df['Buy_Signal'].notna().any():
            apds.append(mpf.make_addplot(df['Buy_Signal'], type='scatter', markersize=50, marker='^', color='g'))

    if not sell_signals.empty:
        sell_dates = pd.to_datetime(sell_signals['Date'])
        df.loc[df.index.isin(sell_dates), 'Sell_Signal'] = df['High'] + (df['High'] - df['Low'])*0.02
        if df['Sell_Signal'].notna().any():
            apds.append(mpf.make_addplot(df['Sell_Signal'], type='scatter', markersize=50, marker='v', color='r'))

    if not close_signals.empty:
        close_dates = pd.to_datetime(close_signals['Date'])
        df.loc[df.index.isin(close_dates), 'Close_Signal'] = df['Close']
        if df['Close_Signal'].notna().any():
            apds.append(mpf.make_addplot(df['Close_Signal'], type='scatter', markersize=100, marker='x', color='k'))

    # Check if apds is empty
    if not apds:
        print(f"No trade signals to plot for {key}.")
    else:
        # Plot price data with signals
        mpf.plot(df, type='candle', style='charles', title=f'{key} Price Chart with Trades', addplot=apds)

# Save Trade Journals
for key, result in results_dict.items():
    trade_journal = result['Trade Journal']
    trade_journal.to_csv(f'Trade_Journal_{key}.csv', index=False)
    print(f"Trade journal for {key} saved.")

# Analyze Trade Journals
for key in results_dict.keys():
    trade_journal = results_dict[key]['Trade Journal']
    print(f"\nTrade Journal for {key}:")
    print(trade_journal)