C:\Users\Administrator\Desktop\transformers2>python MODELPRE.py
Epoch 1/20
100%|██████████████████████████████████████████████████████████████████████████████| 1827/1827 [02:04<00:00, 14.73it/s]
Training losses: {'percent_change': 1.8896324686163053, 'current_leg_change': 0.006542129717471148, 'price': 1959012663.430087}
Training metrics: {'percent_change_acc': 0.19559525643000153, 'current_leg_change_acc': 0.9985454421000394, 'price_mse': 1959012663.430087}
Epoch 2/20
100%|██████████████████████████████████████████████████████████████████████████████| 1827/1827 [01:56<00:00, 15.63it/s]
Training losses: {'percent_change': 1.8867853219368327, 'current_leg_change': 6.649017672368156e-05, 'price': 1873545393.991341}
Training metrics: {'percent_change_acc': 0.19609151736057634, 'current_leg_change_acc': 1.0, 'price_mse': 1873545393.991341}
Epoch 3/20
100%|██████████████████████████████████████████████████████████████████████████████| 1827/1827 [01:56<00:00, 15.65it/s]
Training losses: {'percent_change': 1.9060344828316091, 'current_leg_change': 0.0001647680312719354, 'price': 1741125754.6138234}
Training metrics: {'percent_change_acc': 0.19720382634289918, 'current_leg_change_acc': 0.9999657751082363, 'price_mse': 1741125754.6138234}
Epoch 4/20
100%|██████████████████████████████████████████████████████████████████████████████| 1827/1827 [01:58<00:00, 15.44it/s]
Training losses: {'percent_change': 1.9144180315256523, 'current_leg_change': 0.00016726709475372561, 'price': 1579068039.762753}
Training metrics: {'percent_change_acc': 0.198008111299348, 'current_leg_change_acc': 0.9999486626623544, 'price_mse': 1579068039.762753}
Epoch 5/20
100%|██████████████████████████████████████████████████████████████████████████████| 1827/1827 [01:59<00:00, 15.27it/s]
Training losses: {'percent_change': 1.922026934823145, 'current_leg_change': 1.8179405974034856e-06, 'price': 1388693889.020723}
Training metrics: {'percent_change_acc': 0.19658777829115115, 'current_leg_change_acc': 1.0, 'price_mse': 1388693889.020723}
Epoch 6/20
100%|██████████████████████████████████████████████████████████████████████████████| 1827/1827 [01:59<00:00, 15.31it/s]
Training losses: {'percent_change': 1.942015565847011, 'current_leg_change': 1.6149721064508467e-06, 'price': 1226363228.1312182}
Training metrics: {'percent_change_acc': 0.19530434485000941, 'current_leg_change_acc': 1.0, 'price_mse': 1226363228.1312182}
Epoch 7/20
100%|██████████████████████████████████████████████████████████████████████████████| 1827/1827 [01:58<00:00, 15.38it/s]
Training losses: {'percent_change': 1.9491381018934144, 'current_leg_change': 4.023640462683943e-05, 'price': 1008992092.8945702}
Training metrics: {'percent_change_acc': 0.19496209593237163, 'current_leg_change_acc': 1.0, 'price_mse': 1008992092.8945702}
Epoch 8/20
100%|██████████████████████████████████████████████████████████████████████████████| 1827/1827 [01:58<00:00, 15.37it/s]
Training losses: {'percent_change': 1.9747046936867363, 'current_leg_change': 1.308230559387658e-06, 'price': 942769335.8867841}
Training metrics: {'percent_change_acc': 0.19319951400653695, 'current_leg_change_acc': 1.0, 'price_mse': 942769335.8867841}
Epoch 9/20
100%|██████████████████████████████████████████████████████████████████████████████| 1827/1827 [02:02<00:00, 14.92it/s]
Training losses: {'percent_change': 1.9586877809208227, 'current_leg_change': 0.00018538796388546384, 'price': 710944533.23148}
Training metrics: {'percent_change_acc': 0.19696425210055274, 'current_leg_change_acc': 0.9999657751082363, 'price_mse': 710944533.23148}
Epoch 10/20
100%|██████████████████████████████████████████████████████████████████████████████| 1827/1827 [02:00<00:00, 15.19it/s]
Training losses: {'percent_change': 1.985408586579953, 'current_leg_change': 1.0515825373890723e-08, 'price': 642572836.9541216}
Training metrics: {'percent_change_acc': 0.1937813371665212, 'current_leg_change_acc': 1.0, 'price_mse': 642572836.9541216}
Epoch 11/20
100%|██████████████████████████████████████████████████████████████████████████████| 1827/1827 [02:00<00:00, 15.13it/s]
Training losses: {'percent_change': 1.9766383432523138, 'current_leg_change': 2.876470737571777e-07, 'price': 608595463.3597207}
Training metrics: {'percent_change_acc': 0.19775142461111966, 'current_leg_change_acc': 1.0, 'price_mse': 608595463.3597207}
Epoch 12/20
100%|██████████████████████████████████████████████████████████████████████████████| 1827/1827 [02:01<00:00, 15.04it/s]
Training losses: {'percent_change': 1.9854205040532114, 'current_leg_change': 1.9181645897383734e-08, 'price': 258684971.8231942}
Training metrics: {'percent_change_acc': 0.2002498417098756, 'current_leg_change_acc': 1.0, 'price_mse': 258684971.8231942}
Epoch 13/20
100%|██████████████████████████████████████████████████████████████████████████████| 1827/1827 [02:05<00:00, 14.57it/s]
Training losses: {'percent_change': 2.0099485787812172, 'current_leg_change': 2.785686665030653e-08, 'price': 182091877.0133306}
Training metrics: {'percent_change_acc': 0.1948251963653165, 'current_leg_change_acc': 1.0, 'price_mse': 182091877.0133306}
Epoch 14/20
100%|██████████████████████████████████████████████████████████████████████████████| 1827/1827 [02:09<00:00, 14.14it/s]
Training losses: {'percent_change': 2.016463872341392, 'current_leg_change': 3.6040540148416766e-08, 'price': 73601069.01203005}
Training metrics: {'percent_change_acc': 0.19781987439464724, 'current_leg_change_acc': 1.0, 'price_mse': 73601069.01203005}
Epoch 15/20 import os
import pandas as pd
import numpy as np
import math
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
import torch
from torch.utils.data import Dataset, DataLoader
import torch.nn as nn
import torch.optim as optim
from tqdm import tqdm
from transformers import AutoModel, AutoTokenizer

class TransformerModel(nn.Module):
    def __init__(self, 
                 num_features, 
                 num_cryptos,
                 d_model=128, 
                 nhead=8, 
                 num_encoder_layers=4, 
                 dim_feedforward=512, 
                 dropout=0.1, 
                 num_classes=3,
                 max_seq_length=5000,
                 pretrained_model_name="distilbert-base-uncased"):
        super(TransformerModel, self).__init__()
        
        self.d_model = d_model
        self.crypto_embedding = nn.Embedding(num_cryptos, d_model)
        self.input_linear = nn.Linear(num_features, d_model)
        
        # Load pretrained transformer
        self.pretrained = AutoModel.from_pretrained(pretrained_model_name)
        self.tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name)
        
        #self.positional_encoding = PositionalEncoding(d_model, max_len=max_seq_length)
        
        encoder_layer = nn.TransformerEncoderLayer(d_model=d_model, 
                                                 nhead=nhead, 
                                                 dim_feedforward=dim_feedforward, 
                                                 dropout=dropout,
                                                 batch_first=True)
        
        self.transformer_encoder = nn.TransformerEncoder(encoder_layer, 
                                                       num_layers=num_encoder_layers)
        
        self.dropout = nn.Dropout(dropout)
        
        # Multiple output heads
        self.percent_change_head = nn.Linear(d_model, num_classes)
        self.leg_direction_head = nn.Linear(d_model, 2)  # Binary classification
        self.price_prediction_head = nn.Linear(d_model, 1)  # Regression
    
    def forward(self, src, crypto_id):
        src = self.input_linear(src) * math.sqrt(self.d_model)
        crypto_emb = self.crypto_embedding(crypto_id).unsqueeze(1)
        src = src + crypto_emb
        #src = self.positional_encoding(src)
        
        memory = self.transformer_encoder(src)
        features = memory[:, -1, :]
        features = self.dropout(features)
        
        # Multiple outputs
        percent_change = self.percent_change_head(features)
        leg_direction = self.leg_direction_head(features)
        price_prediction = self.price_prediction_head(features)
        
        return percent_change, leg_direction, price_prediction

class CryptoDataset(Dataset):
    def __init__(self, dataframe, feature_cols, window_size=60):
        self.data = dataframe
        self.feature_cols = feature_cols
        self.window_size = window_size
        self.cryptos = sorted(dataframe['crypto'].unique())
        self.crypto_to_id = {crypto: idx for idx, crypto in enumerate(self.cryptos)}
        self.crypto_data = {
            crypto: dataframe[dataframe['crypto'] == crypto].sort_values('t').reset_index(drop=True)
            for crypto in self.cryptos
        }

        self.indices = []
        for crypto in self.cryptos:
            data_length = len(self.crypto_data[crypto])
            if data_length > self.window_size:
                self.indices.extend([(crypto, idx) for idx in range(data_length - self.window_size)])
    
    def __len__(self):
        return len(self.indices)
    
    def __getitem__(self, idx):
        crypto, seq_start = self.indices[idx]
        data = self.crypto_data[crypto].iloc[seq_start:seq_start + self.window_size]
        features = data[self.feature_cols].values

        # Ensure that target index is within bounds
        target_idx = seq_start + self.window_size
        data_length = len(self.crypto_data[crypto])
        if target_idx >= data_length:
            target_idx = data_length - 1  # Adjust to last index if out of bounds

        # Multiple targets
        percent_change = self.crypto_data[crypto].iloc[target_idx]['percent_change_classification']
        leg_direction = self.crypto_data[crypto].iloc[target_idx]['current_leg_change']
        price = self.crypto_data[crypto].iloc[target_idx]['c']

        crypto_id = self.crypto_to_id[crypto]

        return (
            (torch.tensor(features, dtype=torch.float32), torch.tensor(crypto_id, dtype=torch.long)),
            (
                torch.tensor(percent_change, dtype=torch.long),
                torch.tensor(leg_direction, dtype=torch.long),
                torch.tensor(price, dtype=torch.float32),
            ),
        )
def train(model, dataloader, criterion_dict, optimizer, device):
    model.train()
    epoch_losses = {"percent_change": 0, "current_leg_change": 0, "price": 0}
    metrics = {"percent_change_acc": 0, "current_leg_change_acc": 0, "price_mse": 0}
    total = 0
    
    for (inputs, crypto_ids), (percent_targets, leg_targets, price_targets) in tqdm(dataloader):
        inputs = inputs.to(device)
        crypto_ids = crypto_ids.to(device)
        percent_targets = percent_targets.to(device)
        leg_targets = leg_targets.to(device)
        price_targets = price_targets.to(device)

        optimizer.zero_grad()
        percent_out, leg_out, price_out = model(inputs, crypto_ids)
        
        # Calculate losses
        loss_percent = criterion_dict['classification'](percent_out, percent_targets)
        loss_leg = criterion_dict['classification'](leg_out, leg_targets)
        loss_price = criterion_dict['regression'](price_out.squeeze(), price_targets)
        
        total_loss = loss_percent + loss_leg + loss_price
        total_loss.backward()
        optimizer.step()

        # Update metrics
        batch_size = inputs.size(0)
        total += batch_size
        
        epoch_losses["percent_change"] += loss_percent.item() * batch_size
        epoch_losses["current_leg_change"] += loss_leg.item() * batch_size
        epoch_losses["price"] += loss_price.item() * batch_size
        
        _, predicted_percent = torch.max(percent_out, 1)
        _, predicted_leg = torch.max(leg_out, 1)
        
        metrics["percent_change_acc"] += (predicted_percent == percent_targets).sum().item()
        metrics["current_leg_change_acc"] += (predicted_leg == leg_targets).sum().item()
        metrics["price_mse"] += loss_price.item() * batch_size

    # Normalize metrics
    for key in epoch_losses:
        epoch_losses[key] /= total
    
    metrics["percent_change_acc"] /= total
    metrics["current_leg_change_acc"] /= total
    metrics["price_mse"] /= total
    
    return epoch_losses, metrics



def main():
    # Parameters
    batch_size = 32
    epochs = 20
    learning_rate = 0.001
    window_size = 60  # Sequence length
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    preprocessed_data_dir = 'preprocessed_data'  # Adjust if different

    # Load and concatenate all preprocessed CSV files
    dfs = []
    for ticker in os.listdir(preprocessed_data_dir):
        ticker_dir = os.path.join(preprocessed_data_dir, ticker)
        if os.path.isdir(ticker_dir):
            for file in os.listdir(ticker_dir):
                if file.endswith('_preprocessed.csv'):
                    filepath = os.path.join(ticker_dir, file)
                    df = pd.read_csv(filepath)
                    df['crypto'] = ticker  # Add crypto identifier
                    dfs.append(df)
    if not dfs:
        print("No preprocessed CSV files found.")
        return
    full_df = pd.concat(dfs, ignore_index=True)

    # Define target columns
    target_cols = ['percent_change_classification', 'current_leg_change', 'c']
    required_columns = target_cols + ['t', 'crypto']

    # Verify that all required columns are present
    missing_columns = [col for col in required_columns if col not in full_df.columns]
    if missing_columns:
        print(f"Error: Missing columns in data: {missing_columns}")
        return

    # Define feature columns (all columns except targets and exclusions)
    exclude_cols = required_columns
    feature_cols = [col for col in full_df.columns if col not in exclude_cols]

    # Handle missing values
    full_df.dropna(subset=feature_cols + target_cols, inplace=True)

    # Convert 't' to datetime if not already
    if not pd.api.types.is_datetime64_any_dtype(full_df['t']):
        full_df['t'] = pd.to_datetime(full_df['t'])

    # Sort DataFrame by time
    full_df.sort_values('t', inplace=True)
    full_df.reset_index(drop=True, inplace=True)

    # Split data into training and testing sets
    split_idx = int(len(full_df) * 0.8)
    train_df = full_df.iloc[:split_idx]
    test_df = full_df.iloc[split_idx:]

    # Initialize datasets and dataloaders
    train_dataset = CryptoDataset(train_df, feature_cols, window_size=window_size)
    test_dataset = CryptoDataset(test_df, feature_cols, window_size=window_size)

    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
    test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)

    # Initialize the model
    num_features = len(feature_cols)
    num_cryptos = full_df['crypto'].nunique()
    num_classes = full_df['percent_change_classification'].nunique()

    model = TransformerModel(
        num_features=num_features,
        num_cryptos=num_cryptos,
        d_model=64,
        nhead=4,
        num_encoder_layers=2,
        dim_feedforward=256,
        num_classes=num_classes,
        max_seq_length=window_size,
        pretrained_model_name="distilbert-base-uncased"
    ).to(device)

    # Define loss functions and optimizer
    criterion_dict = {
        'classification': nn.CrossEntropyLoss(),
        'regression': nn.MSELoss()
    }
    optimizer = optim.Adam(model.parameters(), lr=learning_rate)

    # Training loop
    for epoch in range(epochs):
        print(f"Epoch {epoch + 1}/{epochs}")
        epoch_losses, metrics = train(model, train_loader, criterion_dict, optimizer, device)
        print(f"Training losses: {epoch_losses}")
        print(f"Training metrics: {metrics}")

       

    # Save the trained model
    torch.save(model.state_dict(), 'transformer_model.pth')
    print("Model saved as transformer_model.pth")

if __name__ == '__main__':
    main()