How to use an Itarable Dataloader in pytorch?

Marina_Thalassini: <p>I have this code to create my iterable datasets</p>
<pre><code class="lang-auto">import torch
import os
import pickle
import numpy as np
from torch.utils.data import DataLoader, IterableDataset
from datasets import Dataset, load_dataset



feature_columns = [
    'dna_bending_stiffness', 'dna_stacking_energy', 'dna_duplex_free_energy',
    'protein_deformation', 'protein_dna_twist', 'b_dna_twist',
    'dna_bendability', 'nucleosome_positioning', 'z_dna_stability',
    'duplex_disrupt_energy', 'propeller_twist', 'dna_denaturation',
    'dna_a_philicity'
]



# Function to perform one-hot encoding for DNA sequences
def one_hot_encode(sequence):
    encoding = {'A': [1, 0, 0, 0], 'T': [0, 1, 0, 0], 'G': [0, 0, 1, 0], 'C': [0, 0, 0, 1]}
    one_hot_sequence = [encoding.get(base, [0, 0, 0, 0]) for base in sequence]
    return np.array(one_hot_sequence)



def stream_data(absolute_path):
    # Verify if the absolute path exists
    if not os.path.exists(absolute_path):
        raise FileNotFoundError(f"File '{absolute_path}' does not exist.")

    # Use the absolute file path in the data_files dictionary
    data_files = {'train': absolute_path}

    # Load the dataset
    iterable_dataset = load_dataset('csv', data_files=data_files, split='train', streaming=True)

    return iterable_dataset


class TrainDataset:
    def __init__(self, iterable_dataset, test_chromosomes):
        self.iterable_dataset = iterable_dataset
        self.test_chromosomes = test_chromosomes

    def __iter__(self):
        for sample in self.iterable_dataset:
            # Check if the class is not equal to 2
            if sample['class'] != 2 and sample['chrom'] not in self.test_chromosomes:
                yield sample



class TestDataset:
    def __init__(self, iterable_dataset, test_chromosomes):
        self.iterable_dataset = iterable_dataset
        self.test_chromosomes = test_chromosomes

    def __iter__(self):
        # Filter samples for test chromosomes
        class_2_count = 0
        for sample in self.iterable_dataset:
            if sample['chrom'] in self.test_chromosomes:
                if sample['class'] == 2:
                    if class_2_count &lt; class_2_total / 2:
                        sample['class'] = 0
                        yield sample
                        class_2_count += 1
                else:
                    yield sample




class MyIterableTrainDataset(IterableDataset):
    def __init__(self, iterable_dataset, feature_columns):
        self.iterable_dataset = iterable_dataset
        self.feature_columns = feature_columns
        self.counter = 0

    def preprocess_feature(self, feature_values):
        row_matrices = []
        reshaped_feature_data = []
    
        for cell in feature_values:
            # Convert the string representation of the list to a list of floats
            cell_values = [float(value.strip()) for value in cell.strip('[]').split(',')]
            # Convert the list to a numpy array
            cell_array = np.array(cell_values)
            # Reshape the array to (199, 13) matrix
            cell_matrix = cell_array.reshape(199, -1)
            # Append the matrix to the list
            row_matrices.append(cell_matrix)
        # Concatenate the matrices along the second axis to create a single matrix for the row
        row_matrix = np.concatenate(row_matrices, axis=1)
        # Append the row matrix to the list
        reshaped_feature_data.append(row_matrix)
        
            
        return reshaped_feature_data

    def __iter__(self):
        for sample in self.iterable_dataset:
            self.counter += 1

            X1 = one_hot_encode(sample['data'])  # Assuming you have a function one_hot_encode
            X2 = np.array(self.preprocess_feature([sample[col] for col in self.feature_columns]))  # Access individual feature columns
            y = np.array(sample['class'])
    
            X1, X2, y = torch.from_numpy(X1), torch.from_numpy(X2), torch.from_numpy(y)
            
            # Print progress every 10,000 lines
            if self.counter % 10000 == 0:
                
               print("Processed", self.counter, "samples")
             #print(self.counter)  

            yield X1, X2, y ,self.counter


class MyIterableTestDataset(IterableDataset):
    def __init__(self, iterable_dataset, feature_columns):
        self.iterable_dataset = iterable_dataset
        self.feature_columns = feature_columns
        self.counter = 0

    def preprocess_feature(self, feature_values):
        row_matrices = []
        reshaped_feature_data = []
    
        for cell in feature_values:
            # Convert the string representation of the list to a list of floats
            cell_values = [float(value.strip()) for value in cell.strip('[]').split(',')]
            # Convert the list to a numpy array
            cell_array = np.array(cell_values)
            # Reshape the array to (199, 13) matrix
            cell_matrix = cell_array.reshape(199, -1)
            # Append the matrix to the list
            row_matrices.append(cell_matrix)
        # Concatenate the matrices along the second axis to create a single matrix for the row
        row_matrix = np.concatenate(row_matrices, axis=1)
        # Append the row matrix to the list
        reshaped_feature_data.append(row_matrix)
        
            
        return reshaped_feature_data

    def __iter__(self):
        for sample in self.iterable_dataset:
            self.counter += 1
            if self.counter % 10000 == 0:
                print(f"Processed {self.counter} elements")

            X1 = one_hot_encode(sample['data'])  # Assuming you have a function one_hot_encode
            X2 = np.array(self.preprocess_feature([sample[col] for col in self.feature_columns]))  # Access individual feature columns
            y = np.array(sample['class'])
            meta = sample['region']
            X1, X2, y = torch.from_numpy(X1), torch.from_numpy(X2), torch.from_numpy(y)
            #print(X1,X2,y)
            yield X1, X2, y



# Specify the absolute file path
absolute_path = '/home/artemis/combined_200_data_true_features.csv'

# Load the dataset
iterable_dataset = stream_data(absolute_path)

# Define the test chromosomes
test_chromosomes = ['chr1', 'chr2']
valid_chromosomes = ['chr3', 'chr4']



# Create TrainDataset and TestDataset instances
train_dataset = TrainDataset(iterable_dataset, test_chromosomes)
#print(next(iter(train_dataset)))

test_dataset = TestDataset(iterable_dataset, test_chromosomes)
#print(next(iter(test_dataset)))

valid_dataset = TestDataset(iterable_dataset, valid_chromosomes)



train_dataset = MyIterableTrainDataset(train_dataset, feature_columns)
valid_dataset = MyIterableTrainDataset(valid_dataset, feature_columns)
test_dataset = MyIterableTestDataset(test_dataset, feature_columns)



batch_size = 64
# Create train and test data loaders
train_loader = DataLoader(train_dataset, num_workers = 20,  batch_size=batch_size, drop_last = True)
test_loader = DataLoader(test_dataset, num_workers = 20, batch_size=batch_size, drop_last = True)
valid_loader = DataLoader(valid_dataset, num_workers = 20, batch_size=batch_size, drop_last = True)
 



# Directory path
directory = os.path.expanduser("~/loaders_multi")

# Create the directory if it doesn't exist
os.makedirs(directory, exist_ok=True)

# File paths
train_loader_path = os.path.join(directory, 'train_loader.pkl')
test_loader_path = os.path.join(directory, 'test_loader.pkl')
valid_loader_path = os.path.join(directory, 'valid_loader.pkl')

# Save train_loader
with open(train_loader_path, 'wb') as f:
    pickle.dump(train_loader, f)

# Save valid_loader
with open(valid_loader_path, 'wb') as f:
    pickle.dump(test_loader, f)

# Save test_loader
with open(test_loader_path, 'wb') as f:
    pickle.dump(test_loader, f)













</code></pre>
<p>I use only the first 1000 samples just to test</p>
<p>To run my NN i load the dataloaders and i run the following train and test functions:</p>
<pre><code class="lang-auto">def train_step(model: torch.nn.Module,
               dataloader: torch.utils.data.DataLoader,
               loss_fn: torch.nn.Module,
               optimizer: torch.optim.Optimizer,
               device: torch.device) -&gt; Tuple[float, float]:
    """Trains a PyTorch model for a single epoch.

    Turns a target PyTorch model to training mode and then
    runs through all of the required training steps (forward
    pass, loss calculation, optimizer step).

    Args:
    model: A PyTorch model to be trained.
    dataloader: A DataLoader instance for the model to be trained on.
    loss_fn: A PyTorch loss function to minimize.
    optimizer: A PyTorch optimizer to help minimize the loss function.
    device: A target device to compute on (e.g. "cuda" or "cpu").

    Returns:
    A tuple of training loss and training accuracy metrics.
    In the form (train_loss, train_accuracy). For example:

    (0.1112, 0.8743)
    """
    # Put model in train mode
    model.train()

    # Setup train loss and train accuracy values
    train_loss, train_bal_acc, train_mcc, train_f_score, train_auroc = 0, 0, 0, 0, 0

    # Loop through data loader data batches
    for batch, (X1, X2, y) in enumerate(dataloader):
        # Move data to the device if needed
        #X1, X2, y = torch.tensor(X1, dtype=torch.float32), torch.tensor(X2, dtype=torch.float32), torch.tensor(y, dtype=torch.float32)

        
        X1, X2, y = X1.to(device), X2.to(device), y.to(device)

        # 1. Forward pass (model outputs raw logits)
        y_logits = model(X1, X2).squeeze()
        y_pred = torch.round(torch.sigmoid(y_logits))
       
        # Assuming y_pred and threshold are PyTorch tensors
        threshold = torch.tensor(0.5)
        y_pred_class = torch.where(y_pred &gt; threshold, torch.tensor(1), torch.tensor(0))
        
        # 2. Calculate loss/accuracy
        loss = loss_fn(y_logits, y.float())
        train_loss += loss.item()
        
        # 3. Optimizer zero grad
        optimizer.zero_grad()

        # 4. Loss backward
        loss.backward()

        
        # 5. Optimizer step
        optimizer.step()

        y = y.cpu().numpy()
        # get the balanced accuracy
        train_bal_acc += balanced_accuracy_score(y, y_pred_class.cpu().numpy())
        train_mcc += matthews_corrcoef(y, y_pred_class.cpu().numpy())
        train_f_score += f1_score(y, y_pred_class.cpu().numpy(), average='weighted')
        #train_auroc += roc_auc_score(y , y_pred_class.cpu().numpy())

    # Adjust metrics to get average loss and accuracy per batch
    train_loss = train_loss / len(dataloader)
    #train_acc = train_acc / len(dataloader)

    train_bal_acc = train_bal_acc / len(dataloader)
    train_mcc = train_mcc / len(dataloader)
    train_f_score = train_f_score / len(dataloader)
    #train_auroc = train_auroc / len(dataloader)
    
    return train_loss, train_bal_acc, train_mcc, train_f_score

def test_step(model: torch.nn.Module,
              dataloader: torch.utils.data.DataLoader,
              loss_fn: torch.nn.Module,
              device: torch.device) -&gt; Tuple[float, float]:
    """Tests a PyTorch model for a single epoch.

    Turns a target PyTorch model to "eval" mode and then performs
    a forward pass on a testing dataset.

    Args:
    model: A PyTorch model to be tested.
    dataloader: A DataLoader instance for the model to be tested on.
    loss_fn: A PyTorch loss function to calculate loss on the test data.
    device: A target device to compute on (e.g. "cuda" or "cpu").

    Returns:
    A tuple of testing loss and testing accuracy metrics.
    In the form (test_loss, test_accuracy). For example:

    (0.0223, 0.8985)
    """
    # Put model in eval mode
    model.eval()

    # Setup test loss and test accuracy values
    test_loss, test_bal_acc, test_mcc, test_f_score, test_auroc = 0, 0, 0, 0, 0
    total = 0

    # Turn on inference context manager
    with torch.inference_mode():
        # Loop through DataLoader batches
        for batch, (X1, X2 ,y) in enumerate(dataloader):
            #X1, X2, y = torch.tensor(X1, dtype=torch.float32), torch.tensor(X2, dtype=torch.float32), torch.tensor(y, dtype=torch.float32)
         

            # Send data to target device
            X1, X2, y = X1.to(device).float(), X2.to(device).float(), y.to(device).float()

            # 1. Forward pass
            y_logits = model(X1, X2).squeeze()
            test_pred = torch.round(torch.sigmoid(y_logits))
            threshold = torch.tensor(0.5)
            test_pred_labels = torch.where(test_pred &gt; threshold, torch.tensor(1), torch.tensor(0)) 

            # 2. Calculate and accumulate loss
            loss = loss_fn(y_logits, y)
            test_loss += loss.item()

            total += y.size(0)
            print(f"total = {total}")
            # Calculate and accumulate accuracy
            y = y.cpu().numpy()
            # get the balanced accuracy
            test_bal_acc += balanced_accuracy_score(y, test_pred_labels.cpu().numpy())
            test_mcc += matthews_corrcoef(y, test_pred_labels.cpu().numpy())
            test_f_score += f1_score(y, test_pred_labels.cpu().numpy(), average='weighted')
          #  test_auroc += roc_auc_score(y , test_pred_labels.cpu().numpy())

    # Adjust metrics to get average loss and accuracy per batch
    test_loss = test_loss / total
    #test_acc = test_acc / len(dataloader)
    test_bal_acc = test_bal_acc /total
    test_mcc = test_mcc  / total
    test_f_score = test_f_score / total
 # test_auroc = test_auroc / len(dataloader)
    return test_loss, test_bal_acc, test_mcc, test_f_score

</code></pre>
<p>which are integrated into a single function with early stopping applied</p>
<p>I am not if i calculate the test metrics rights. Because I dont have access to the whole len of the dataloader I have to calculate it with a loop using :total += y.size(0).</p>
<p>However i don’t seem to get right results, loss is around 0.001 but the same goes with acc,f_score and the rest.</p>
<p>I am not sure how to calculate the test metrics.</p>
<p>Is this the correct way to pass my data in the forward function ?</p>