output for : Preparing sequences...
Shape of X: (887, 60, 10)
Shape of Y[percent_change_classification]: (887,)
Shape of Y[leg_direction]: (887,)
Verifying label ranges...
percent_change_classification labels: min=0, max=6
leg_direction labels: min=0, max=1
Splitting data into training and testing sets...
Unique labels for 'percent_change_classification': [0 1 2 3 4 5 6]
Sample labels: [1 2 0 4 5 4 3 1 3 1]
Unique labels for 'leg_direction': [0 1]
Sample labels: [0 0 0 1 1 1 0 0 0 0]
Building the Transformer model...
2024-11-07 13:23:01.587167: E external/local_xla/xla/stream_executor/cuda/cuda_driver.cc:152] failed call to cuInit: INTERNAL: CUDA error: Failed call to cuInit: UNKNOWN ERROR (303)
Model Output Names: ListWrapper(['percent_change_classification', 'leg_direction'])
Model Output Names: ListWrapper(['percent_change_classification', 'leg_direction'])
Model Summary:
Model: "functional"
┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┓
┃ Layer (type) ┃ Output Shape ┃ Param # ┃ Connected to ┃
┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━┩
│ input (InputLayer) │ (None, 60, 10) │ 0 │ - │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ add (Add) │ (None, 60, 10) │ 0 │ input[0][0] │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ layer_normalization │ (None, 60, 10) │ 20 │ add[0][0] │
│ (LayerNormalizatio… │ │ │ │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ multi_head_attenti… │ (None, 60, 10) │ 44,042 │ layer_normalizat… │
│ (MultiHeadAttentio… │ │ │ layer_normalizat… │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ dropout_1 (Dropout) │ (None, 60, 10) │ 0 │ multi_head_atten… │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ add_1 (Add) │ (None, 60, 10) │ 0 │ add[0][0], │
│ │ │ │ dropout_1[0][0] │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ layer_normalizatio… │ (None, 60, 10) │ 20 │ add_1[0][0] │
│ (LayerNormalizatio… │ │ │ │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ dense_1 (Dense) │ (None, 60, 128) │ 1,408 │ layer_normalizat… │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ dropout_2 (Dropout) │ (None, 60, 128) │ 0 │ dense_1[0][0] │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ dense_2 (Dense) │ (None, 60, 10) │ 1,290 │ dropout_2[0][0] │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ dropout_3 (Dropout) │ (None, 60, 10) │ 0 │ dense_2[0][0] │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ add_2 (Add) │ (None, 60, 10) │ 0 │ add_1[0][0], │
│ │ │ │ dropout_3[0][0] │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ layer_normalizatio… │ (None, 60, 10) │ 20 │ add_2[0][0] │
│ (LayerNormalizatio… │ │ │ │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ multi_head_attenti… │ (None, 60, 10) │ 44,042 │ layer_normalizat… │
│ (MultiHeadAttentio… │ │ │ layer_normalizat… │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ dropout_5 (Dropout) │ (None, 60, 10) │ 0 │ multi_head_atten… │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ add_3 (Add) │ (None, 60, 10) │ 0 │ add_2[0][0], │
│ │ │ │ dropout_5[0][0] │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ layer_normalizatio… │ (None, 60, 10) │ 20 │ add_3[0][0] │
│ (LayerNormalizatio… │ │ │ │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ dense_3 (Dense) │ (None, 60, 128) │ 1,408 │ layer_normalizat… │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ dropout_6 (Dropout) │ (None, 60, 128) │ 0 │ dense_3[0][0] │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ dense_4 (Dense) │ (None, 60, 10) │ 1,290 │ dropout_6[0][0] │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ dropout_7 (Dropout) │ (None, 60, 10) │ 0 │ dense_4[0][0] │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ add_4 (Add) │ (None, 60, 10) │ 0 │ add_3[0][0], │
│ │ │ │ dropout_7[0][0] │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ global_average_poo… │ (None, 10) │ 0 │ add_4[0][0] │
│ (GlobalAveragePool… │ │ │ │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ dropout_8 (Dropout) │ (None, 10) │ 0 │ global_average_p… │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ dense_5 (Dense) │ (None, 128) │ 1,408 │ dropout_8[0][0] │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ dropout_9 (Dropout) │ (None, 128) │ 0 │ dense_5[0][0] │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ percent_change_cla… │ (None, 7) │ 903 │ dropout_9[0][0] │
│ (Dense) │ │ │ │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ leg_direction │ (None, 1) │ 129 │ dropout_9[0][0] │
│ (Dense) │ │ │ │
└─────────────────────┴───────────────────┴────────────┴───────────────────┘
Total params: 96,000 (375.00 KB)
Trainable params: 96,000 (375.00 KB)
Non-trainable params: 0 (0.00 B)
None
you can see x and y are 1d but the model architecture is wrong i think import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import pickle
import logging
from ta.volatility import AverageTrueRange, BollingerBands
from ta.momentum import RSIIndicator
from ta.trend import MACD, ADXIndicator
from sklearn.preprocessing import LabelEncoder, StandardScaler
from sklearn.model_selection import train_test_split
from tensorflow.keras.layers import (
    Input,
    Dense,
    Dropout,
    MultiHeadAttention,
    LayerNormalization,
    GlobalAveragePooling1D,
)
from tensorflow.keras.models import Model
import tensorflow as tf
from binance.client import Client

# Suppress TensorFlow warnings for clarity
logging.getLogger("tensorflow").setLevel(logging.ERROR)

def load_data(client, symbol, interval='1m', limit=1000):
    """
    Fetch historical kline data for a given symbol.
    """
    try:
        historicals = client.get_klines(symbol=symbol, interval=interval, limit=limit)
        data = []
        for kl in historicals:
            open_time = pd.to_datetime(kl[0], unit='ms')
            open_price = float(kl[1])
            high = float(kl[2])
            low = float(kl[3])
            close = float(kl[4])
            volume = float(kl[5])
            data.append([open_time, open_price, high, low, close, volume])
        df = pd.DataFrame(data, columns=['open_time', 'open', 'high', 'low', 'close', 'volume'])
        df.set_index('open_time', inplace=True)
        return df
    except Exception as e:
        print(f"Error fetching data for {symbol}: {e}")
        return pd.DataFrame()

def add_technical_indicators(df):
    """
    Add technical indicators to the DataFrame.
    """
    # RSI
    rsi = RSIIndicator(close=df['close'], window=14)
    df['RSI'] = rsi.rsi()
    
    # MACD
    macd = MACD(close=df['close'], window_slow=26, window_fast=12, window_sign=9)
    df['MACD'] = macd.macd()
    df['MACD_signal'] = macd.macd_signal()
    df['MACD_diff'] = macd.macd_diff()
    
    # ATR
    atr = AverageTrueRange(high=df['high'], low=df['low'], close=df['close'], window=14)
    df['ATR'] = atr.average_true_range()
    
    # Bollinger Bands Width
    bollinger = BollingerBands(close=df['close'], window=20, window_dev=2)
    df['BB_upper'] = bollinger.bollinger_hband()
    df['BB_lower'] = bollinger.bollinger_lband()
    df['BB_width'] = (df['BB_upper'] - df['BB_lower']) / df['close']
    
    # ADX
    adx = ADXIndicator(high=df['high'], low=df['low'], close=df['close'], window=14)
    df['ADX'] = adx.adx()
    
    # Drop initial rows with NaN values
    df.dropna(inplace=True)
    return df

def classify_market_environments(df):
    """
    Classify market environments into numerical categories for model training.
    """
    # Volatility Classification
    df['ATR_mavg'] = df['ATR'].rolling(window=14).mean()
    df['ATR_vol'] = np.where(df['ATR'] > 1.2 * df['ATR_mavg'], 'High',
                             np.where(df['ATR'] < 0.8 * df['ATR_mavg'], 'Low', 'Medium'))
    
    df['BB_mavg'] = df['BB_width'].rolling(window=20).mean()
    df['BB_vol'] = np.where(df['BB_width'] > 1.2 * df['BB_mavg'], 'High',
                            np.where(df['BB_width'] < 0.8 * df['BB_mavg'], 'Low', 'Medium'))
    
    df['daily_return'] = df['close'].pct_change()
    df['RV'] = df['daily_return'].rolling(window=20).std() * np.sqrt(252)
    rv_80 = df['RV'].quantile(0.8)
    rv_20 = df['RV'].quantile(0.2)
    df['RV_vol'] = np.where(df['RV'] > rv_80, 'High',
                            np.where(df['RV'] < rv_20, 'Low', 'Medium'))
    
    df['Volatility'] = df[['ATR_vol', 'BB_vol', 'RV_vol']].mode(axis=1)[0]
    
    df['Trend'] = np.where(
        (df['MACD'] > df['MACD_signal']) & (df['MACD'] > 0), 'Bullish',
        np.where(
            (df['MACD'] < df['MACD_signal']) & (df['MACD'] < 0), 'Bearish', 'Neutral'
        )
    )
    
    df['Trend_strength'] = np.where(df['ADX'] > 25, 'Strong', 'Weak')
    
    df['Market_Environment'] = df.apply(
        lambda row: f"{row['Volatility']} Vol/{row['Trend']}" if row['Trend_strength'] == 'Strong' else f"{row['Volatility']} Vol/Neutral",
        axis=1
    )

    # Encode categorical data to numeric
    label_encoders = {}
    for column in ['ATR_vol', 'BB_vol', 'RV_vol', 'Volatility', 'Trend', 'Trend_strength', 'Market_Environment']:
        le = LabelEncoder()
        df[column] = le.fit_transform(df[column].astype(str))
        label_encoders[column] = le  # Save encoder for each column if you need to decode later

    return df, label_encoders  # Returning encoders in case needed

def calculate_leg_data(df):
    """
    Calculate leg data: current and previous leg percent change and length.
    """
    df['percent_delta'] = df['close'].pct_change()
    df = df.reset_index(drop=True)
    
    previous_leg_change = 0
    previous_leg_length = 0
    current_leg_change = 0
    current_leg_length = 0
    
    previous_changes = []
    previous_lengths = []
    current_changes = []
    current_lengths = []
    
    for i in range(len(df)):
        if i == 0:
            current_leg_change = 0
            current_leg_length = 0
        else:
            percent_delta = df.at[i, 'percent_delta']
            if current_leg_length == 0:
                current_leg_change = percent_delta
                current_leg_length = 1
            else:
                if (current_leg_change > 0 and percent_delta > 0) or (current_leg_change < 0 and percent_delta < 0):
                    current_leg_change += percent_delta
                    current_leg_length += 1
                else:
                    previous_leg_change = current_leg_change
                    previous_leg_length = current_leg_length
                    current_leg_change = percent_delta
                    current_leg_length = 1
        previous_changes.append(previous_leg_change)
        previous_lengths.append(previous_leg_length)
        current_changes.append(current_leg_change)
        current_lengths.append(current_leg_length)
    
    df['previous_leg_change'] = previous_changes
    df['previous_leg_length'] = previous_lengths
    df['current_leg_change'] = current_changes
    df['current_leg_length'] = current_lengths
    
    df.drop(columns=['percent_delta'], inplace=True)
    return df

def classify_percent_change(df):
    """
    Classify percent changes into 7 categories.
    """
    df['percent_change'] = df['close'].pct_change()
    df.dropna(inplace=True)
    
    percentiles = df['percent_change'].quantile([0.05, 0.20, 0.40, 0.60, 0.80, 0.95]).to_dict()
    
    def classify(x, p):
        if x < p[0.05]:
            return 'Down a Lot'
        elif x < p[0.20]:
            return 'Down Moderate'
        elif x < p[0.40]:
            return 'Down a Little'
        elif x < p[0.60]:
            return 'No Change'
        elif x < p[0.80]:
            return 'Up a Little'
        elif x < p[0.95]:
            return 'Up Moderate'
        else:
            return 'Up a Lot'
    
    df['percent_change_classification'] = df['percent_change'].apply(lambda x: classify(x, percentiles))
    return df

def encode_and_scale(df, label_encoders=None, scalers=None):
    """
    Encode categorical features and scale numerical features.
    """
    if label_encoders is None:
        label_encoders = {}
    if scalers is None:
        scalers = {}
    
    # 1. Encode Categorical Input Features
    categorical_features = ['percent_change_classification', 'Market_Environment']
    for col in categorical_features:
        le = LabelEncoder()
        df[f'{col}_encoded'] = le.fit_transform(df[col].astype(str))
        label_encoders[col] = le  # Save the encoder for future use
    
    # 2. Encode Categorical Target Features (if applicable)
    # In this case, 'leg_direction' is binary (0 or 1), so no encoding is needed.
    
    # 3. Scale Numerical Features
    numerical_features = [
        'RSI', 'MACD', 'MACD_signal', 'ATR', 'BB_width',
        'previous_leg_change', 'previous_leg_length',
        'current_leg_change', 'current_leg_length'
    ]
    
    scaler = StandardScaler()
    df[numerical_features] = scaler.fit_transform(df[numerical_features])
    scalers['numerical'] = scaler  # Save the scaler for future use
    
    # 4. Scale Regression Targets
    # For this two-output model, we're excluding regression targets.
    # If needed in the future, scaling can be applied here.
    
    return df, label_encoders, scalers

def prepare_sequences(df, seq_length=60, target_steps=1):
    """
    Prepare input sequences and corresponding multiple targets.

    Returns:
    - X (np.ndarray): Input sequences.
    - Y (dict): Dictionary containing multiple targets.
    """
    FEATURES = [
        'RSI', 'MACD', 'MACD_signal', 'ATR', 'BB_width',
        'previous_leg_change', 'previous_leg_length',
        'current_leg_change', 'current_leg_length',
        'Market_Environment_encoded'
    ]
    
    X = []
    Y_percent_change = []
    Y_leg_direction = []
    
    for i in range(len(df) - seq_length - target_steps + 1):
        seq = df.iloc[i:i + seq_length][FEATURES].values
        target_percent_change = df.iloc[i + seq_length]['percent_change_classification_encoded']
        target_leg_direction = 1 if df.iloc[i + seq_length]['current_leg_change'] > 0 else 0
        
        X.append(seq)
        Y_percent_change.append(target_percent_change)
        Y_leg_direction.append(target_leg_direction)
    
    X = np.array(X)
    Y = {
        'percent_change_classification': np.array(Y_percent_change),
        'leg_direction': np.array(Y_leg_direction)
    }
    
    return X, Y

def build_multi_output_transformer_model(
    input_shape, 
    num_classes_classification, 
    head_size=256, 
    num_heads=4, 
    ff_dim=128, 
    num_transformer_blocks=2, 
    dropout=0.1
):
    """
    Builds a Transformer model with multiple outputs.
    """
    inputs = Input(shape=input_shape, name='input')

    # Positional Encoding (Learnable)
    position_indices = tf.range(start=0, limit=input_shape[0], delta=1)
    positional_embedding = Dense(input_shape[1])(tf.expand_dims(position_indices, axis=1))
    x = inputs + positional_embedding  # Broadcasting addition

    # Transformer Encoder Blocks
    for _ in range(num_transformer_blocks):
        # Layer Normalization 1
        x_norm = LayerNormalization(epsilon=1e-6)(x)
        
        # Multi-Head Attention
        attention = MultiHeadAttention(key_dim=head_size, num_heads=num_heads, dropout=dropout)(x_norm, x_norm)
        attention = Dropout(dropout)(attention)
        x = x + attention  # Residual connection

        # Layer Normalization 2
        x_norm = LayerNormalization(epsilon=1e-6)(x)
        
        # Feed-Forward Network
        ffn = Dense(ff_dim, activation="relu")(x_norm)
        ffn = Dropout(dropout)(ffn)
        ffn = Dense(input_shape[-1])(ffn)
        ffn = Dropout(dropout)(ffn)
        x = x + ffn  # Residual connection

    # Global Average Pooling
    x = GlobalAveragePooling1D()(x)
    x = Dropout(dropout)(x)
    x = Dense(128, activation="relu")(x)
    x = Dropout(dropout)(x)

    # Output: Percent Change Classification (7 classes)
    percent_change_output = Dense(num_classes_classification, activation='softmax', name='percent_change_classification')(x)

    # Output: Leg Direction (Binary Classification)
    leg_direction_output = Dense(1, activation='sigmoid', name='leg_direction')(x)  # Changed to sigmoid

    # Define the Model
    model = Model(
        inputs=inputs, 
        outputs=[
            percent_change_output, 
            leg_direction_output
        ]
    )
    
    return model

def compile_multi_output_model(model):
    """
    Compiles the multi-output Transformer model with appropriate loss functions and metrics.
    """
    model.compile(
        optimizer='adam',
        loss={
            'percent_change_classification': 'sparse_categorical_crossentropy',
            'leg_direction': 'binary_crossentropy'
        },
        metrics={
            'percent_change_classification': ['accuracy'],
            'leg_direction': ['accuracy']
        }
    )
    # After building and compiling the model
    print("Model Output Names:", model.output_names)

    return model

def plot_attention_matrices(model, sample_input, seq_length, input_dim):
    """
    Visualize attention matrices for a sample input.
    Note: Accessing exact attention weights requires custom layers or callbacks.
    Here, we'll provide a simplified visualization.
    """
    # Get the first transformer block's attention layer
    attention_layer = None
    for layer in model.layers:
        if isinstance(layer, MultiHeadAttention):
            attention_layer = layer
            break
    
    if attention_layer is None:
        print("No MultiHeadAttention layer found in the model.")
        return
    
    # Create a temporary model to get attention outputs
    attention_model = Model(inputs=model.input, outputs=attention_layer.output)
    attention_output = attention_model.predict(sample_input[np.newaxis, ...])
    
    # Since TensorFlow doesn't expose attention weights directly, we'll visualize the output.
    # For detailed attention weights, consider using custom training loops or TensorFlow callbacks.
    plt.figure(figsize=(10, 6))
    sns.heatmap(attention_output[0].mean(axis=-1), cmap='viridis')
    plt.title('Attention Output Heatmap (Mean over Feature Dimensions)')
    plt.xlabel('Sequence Position')
    plt.ylabel('Sequence Step')
    plt.show()

def main():
    # Configuration
    BINANCE_API_KEY = 'YOUR_API_KEY'          # Replace with your API key
    BINANCE_API_SECRET = 'YOUR_API_SECRET'    # Replace with your API secret
    
    SYMBOL = 'BTCUSDT'      # You can change this to your desired symbol
    INTERVAL = '1m'         # Changed to '1m' for practicality; '1s' may cause rate limits
    SEQ_LENGTH = 60         # Number of time steps in each input sequence
    TARGET_STEPS = 1        # Predicting the next time step
    LIMIT = 2000            # Number of historical data points to fetch per symbol (increased for better training)
    
    # Initialize Binance client
    client = Client(BINANCE_API_KEY, BINANCE_API_SECRET)
    
    # Load and prepare data
    print("Loading data...")
    df = load_data(client, SYMBOL, interval=INTERVAL, limit=LIMIT)
    if df.empty:
        print("No data fetched. Exiting.")
        return
    print("Adding technical indicators...")
    df = add_technical_indicators(df)
    print("Classifying market environments...")
    df, market_env_encoders = classify_market_environments(df)
    print("Calculating leg data...")
    df = calculate_leg_data(df)
    print("Classifying percent changes...")
    df = classify_percent_change(df)
    
    # Encode and scale data
    print("Encoding and scaling data...")
    df, label_encoders, scalers = encode_and_scale(df)
    
    # Prepare sequences with two targets
    print("Preparing sequences...")
    X, Y = prepare_sequences(df, SEQ_LENGTH, TARGET_STEPS)
    
    # Validate data shapes
    print(f"Shape of X: {X.shape}")
    for key in Y:
        print(f"Shape of Y[{key}]: {Y[key].shape}")
    
    # Check if data is sufficient
    if len(X) == 0:
        print("Insufficient data after preparing sequences.")
        return
    
    # Unpack Y dictionary
    Y_percent_change = Y['percent_change_classification']
    Y_leg_direction = Y['leg_direction']
    
    # Assert consistent sample sizes
    assert X.shape[0] == Y_percent_change.shape[0] == Y_leg_direction.shape[0], "Mismatch in sample sizes between X and Y."
    
    # Verify label ranges
    print("Verifying label ranges...")
    print(f"percent_change_classification labels: min={Y_percent_change.min()}, max={Y_percent_change.max()}")
    print(f"leg_direction labels: min={Y_leg_direction.min()}, max={Y_leg_direction.max()}")
    
    assert Y_percent_change.min() >= 0 and Y_percent_change.max() < 7, "percent_change_classification labels out of range [0,6]"
    assert Y_leg_direction.min() >= 0 and Y_leg_direction.max() <= 1, "leg_direction labels should be binary (0 or 1)"
    
    # Split data
    print("Splitting data into training and testing sets...")
    X_train, X_test, y_percent_change_train, y_percent_change_test, y_leg_direction_train, y_leg_direction_test = train_test_split(
        X,
        Y_percent_change,
        Y_leg_direction,
        test_size=0.2,
        random_state=42
    )
    
    # Reassemble Y_train and Y_test dictionaries
    Y_train = {
        'percent_change_classification': y_percent_change_train,
        'leg_direction': y_leg_direction_train
    }
    
    Y_test = {
        'percent_change_classification': y_percent_change_test,
        'leg_direction': y_leg_direction_test
    }
    # Check unique labels and sample labels
    print("Unique labels for 'percent_change_classification':", np.unique(Y_train['percent_change_classification']))
    print("Sample labels:", Y_train['percent_change_classification'][:10])
    print("Unique labels for 'leg_direction':", np.unique(Y_train['leg_direction']))
    print("Sample labels:", Y_train['leg_direction'][:10])
    # Inspect loss functions

    # Build and compile model
    print("Building the Transformer model...")
    input_shape = (SEQ_LENGTH, X.shape[2])
    num_classes = len(np.unique(Y_percent_change))  # Should be 7
    model = build_multi_output_transformer_model(input_shape, num_classes)
    model = compile_multi_output_model(model)
      # After building and compiling the model
    print("Model Output Names:", model.output_names)
    # Print model summary
    print("Model Summary:")
    print(model.summary())
 
    # Train model
    print("Training the model...")
    print(Y_train['percent_change_classification'].shape)
    history = model.fit(
        X_train,
        {
            'percent_change_classification': Y_train['percent_change_classification'],
            'leg_direction': Y_train['leg_direction']
        },
        epochs=10,
        batch_size=32,
        validation_split=0.2
    )
    
    # Save encoders and scalers for future use
    print("Saving label encoders and scalers...")
    with open('label_encoders.pkl', 'wb') as f:
        pickle.dump(label_encoders, f)
    
    with open('scalers.pkl', 'wb') as f:
        pickle.dump(scalers, f)
    
    # Save the model
    print("Saving the trained model...")
    model.save('multi_output_transformer_model.h5')
    
    # Visualize training history
    print("Plotting training history...")
    plt.figure(figsize=(12, 4))
    
    # Plot loss
    plt.subplot(1, 2, 1)
    for key in history.history:
        if 'loss' in key and key != 'loss':
            plt.plot(history.history[key], label=key)
    plt.title('Training Loss')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.legend()
    
    # Plot accuracy
    plt.subplot(1, 2, 2)
    for key in history.history:
        if 'accuracy' in key:
            plt.plot(history.history[key], label=key)
    plt.title('Training Accuracy')
    plt.xlabel('Epoch')
    plt.ylabel('Accuracy')
    plt.legend()
    
    plt.tight_layout()
    plt.show()
    
    # Visualization of attention matrices (simplified)
    print("Visualizing attention matrices for a sample input...")
    sample_input = X_test[0]
    plot_attention_matrices(model, sample_input, SEQ_LENGTH, X.shape[2])
    
    print("Training and visualization complete.")
    return model, history

if __name__ == "__main__":
    model, history = main() SIMPLYFY IT WE CAN ADD COMPLIEXITY ALTER