LSTM Issue with memory train/validation need help

John_Doe2: <p>I have a problem. when i change the order of the hyperparameter values in the for loops, the results change of both train and validation.<br>
to clarify: same hyperparameter values result different test/validation depending if they were iterated first or later.</p>
<p>it is not about random state or different data. as i initialize with same. can you see what is causing the test/validation loop to effect the model training and test/validation in next iterations? if i remove the validation loop, the training will be ok also when i change order of the hyperparameters:</p>
<p>so removing the last for loop for validation, which is just few lines of code removes the problem. but i need that loop to check if training is converging on validation data.</p>
<pre><code class="lang-auto">'''imports'''
torch.manual_seed(42)  # Set the seed value for PyTorch
torch.backends.cudnn.deterministic = True  # If using CUDA for GPU acceleration
torch.backends.cudnn.benchmark = False  # Disable CuDNN benchmarking for determinism
'''some code inbetween loading data and features'''
    for seq_length in [10,20,50]:
        input_size = len(features)
        for dropout_prob in [0.3,0.2]:
            for hidden_size in [5]:
                for num_layers in [3]:
                    for num_epochs in [2]:
                        for learning_rate_value in [0.0001,0.01]:
                            for batch_size in [100]:
                                for weight_decay_value in [0.5,0]:
                                    run += 1
                                    torch.cuda.empty_cache()
                                    gc.collect()

                                    for obj in gc.get_objects():
                                        try:

                                            if str(obj).startswith("&lt;class 'torch."):

                                                del obj
                                        except:
                                            pass
                                    allocated_memory = torch.cuda.memory_allocated()

                                    # Print the allocated memory
                                    print(allocated_memory)

                                    data = df[features + target].copy().values.astype(float)

                                    # Normalize the features
                                    feature_scaler = MinMaxScaler()
                                    data[:, :-1] = feature_scaler.fit_transform(data[:, :-1])

                                    # Normalize the target ('Close') separately
                                    target_scaler = MinMaxScaler()
                                    data[:, -1:] = target_scaler.fit_transform(data[:, -1:])
                                    sequences, targets = create_sequences(data, seq_length)

                                    # Split the data into train and test sets
                                    split_percentage = 0.8
                                    holdout_percentage = 0.9
                                    split_index = int(len(sequences) * split_percentage)
                                    holdout_index = int(len(sequences) * holdout_percentage)

                                    train_sequences = sequences[:split_index]
                                    train_targets = targets[:split_index]
                                    test_sequences = sequences[split_index:holdout_index]
                                    test_targets = targets[split_index:holdout_index]

                                    # Convert data to PyTorch tensors
                                    train_sequences_tensor = torch.Tensor(train_sequences).to(device)
                                    train_targets_tensor = torch.Tensor(train_targets).to(device)
                                    test_sequences_tensor = torch.Tensor(test_sequences).to(device)
                                    test_targets_tensor = torch.Tensor(test_targets).to(device)
                                    full_sequences_tensor = torch.Tensor(sequences).to(device)
                                    full_targets_tensor = torch.Tensor(targets).to(device)

                                    train_data = TensorDataset(train_sequences_tensor, train_targets_tensor)
                                    test_data = TensorDataset(test_sequences_tensor, test_targets_tensor)
                                    full_data = TensorDataset(full_sequences_tensor, full_targets_tensor)

                                    # Create data loaders

                                    train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True, num_workers=0)
                                    test_loader = DataLoader(test_data, batch_size=1, num_workers=0)
                                    full_loader = DataLoader(full_data, batch_size=batch_size, num_workers=0)

                                    # Initialize the model and optimizer
                                    model = LSTMModel(input_size, hidden_size, num_layers,dropout_prob).to(device)
                                    criterion = nn.SmoothL1Loss()
                                    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate_value,weight_decay=weight_decay_value)
                                    optimizer.zero_grad()

                                    best_loss = float('inf')  # Initialize best loss as positive infinity
                                    early_stopping_counter = 0  # Initialize early stopping counter
                                    losses = []
                                    # Training loop
                                    previous_average_loss = float('inf')
                                    best_rmse_this_set = float('inf')

                                   
                                    best_epoch_for_this_set = 0
                                    previous_rmse = float('inf')
                                    print()
                                    print("Start run", run, "params ", seq_length, hidden_size, num_layers,
                                          num_epochs, learning_rate_value, batch_size, weight_decay_value,
                                          dropout_prob)
                                    correl_counter = 0
                                    correl_streak =0
                                    previous_correl = 0
                                    loss = 0
                                    for epoch in range(num_epochs):


                                        model.train()

                                        for inputs, targets in train_loader:

                                            optimizer.zero_grad()
                                            outputs = model(inputs)
                                            targets = targets.view_as(outputs)
                                            loss = custom_loss(outputs, targets)

                                            losses.append(loss.item())
                                            loss.backward()
                                            optimizer.step()


                                   
                                        with torch.no_grad():
                                            average_loss = np.mean(losses)
                                            #print(average_loss)


                                        with torch.no_grad():
                                            model.eval()
                                            criteriontest = nn.SmoothL1Loss()
                                            test_losses = []
                                            test_loss = 0
                                            training_predicted_prices = []
                                            all_predictions = []
                                            all_targets = []
                                            for inputstest, targetstest in test_loader:
                                                model.eval()
                                                outputsz = model(inputstest)
                                                targetstest = targetstest.view_as(outputsz)
                                                test_loss = criteriontest(outputsz, targetstest)
                                                test_losses.append(test_loss.item())
</code></pre>