Issue: mini-batch GD with batch_size=1 should give same metrics as SGD, but that is not happening here:

class DenseNeuralNetwork:
    def __init__(self, layer_sizes, activations, loss_f):
        self.learning_rate = None
        self.layer_sizes = layer_sizes
        self.loss_f = loss_f

        np.random.seed(42)
        self.W = [None] + [np.random.uniform(-1, 1, (layer_sizes[i], layer_sizes[i-1])) 
                           for i in range(1, len(layer_sizes))]
        self.b = [None] + [np.random.uniform(-1, 1, layer_sizes[i]) 
                           for i in range(1, len(layer_sizes))]

        self.a = [np.zeros(size) for size in layer_sizes]  # Activations
        self.z = [None] + [np.zeros(layer_sizes[i]) for i in range(1, len(layer_sizes))]  # Net inputs

        self.activation_f = activations  # Activation functions for each layer

    def propagate_forward(self, x):
        self.a[0] = x  # Input layer activation

        for i in range(1, len(self.W)):
            self.z[i] = self.W[i] @ self.a[i-1] + self.b[i]  # Weighted sum
            self.a[i] = self.activation_f[i].function(self.z[i])  # Activation

    def propagate_backward(self, y_true):
        error_term = {}
        dW = [None] + [np.zeros_like(w) for w in self.W[1:]]  # Gradient accumulators for weights
        db = [None] + [np.zeros_like(b) for b in self.b[1:]]  # Gradient accumulators for biases

        L = len(self.W) - 1  # Index of the last layer

        # Compute error term for the output layer
        if self.activation_f[L] == Softmax:
            # For Softmax with Categorical Cross-Entropy loss
            error_term[L] = self.activation_f[L].derivative(self.z[L]) @ self.loss_f.gradient(y_true, self.a[L])
        else:
            error_term[L] = self.activation_f[L].derivative(self.z[L]) * self.loss_f.gradient(y_true, self.a[L])

        # Compute gradients for the output layer
        dW[L] = np.outer(error_term[L], self.a[L-1])  # Shape: (output_size, prev_layer_size)
        db[L] = error_term[L]  # Shape: (output_size,)

        # Compute error and gradients for hidden layers
        for i in reversed(range(1, L)):
            if self.activation_f[i] == Softmax:
                # For softmax activation
                error_term[i] = self.activation_f[i].derivative(self.z[i]) @ (self.W[i+1].T @ error_term[i+1])
            else:
                # For ReLU and others
                error_term[i] = self.activation_f[i].derivative(self.z[i]) * (self.W[i+1].T @ error_term[i+1])

            # Compute gradients for layer i
            dW[i] = np.outer(error_term[i], self.a[i-1])  # Shape: (current_layer_size, prev_layer_size)
            db[i] = error_term[i]  # Shape: (current_layer_size,)

        return dW, db  # Return gradients

    def train_sgd(self, X_train, y_train, X_val, y_val, epochs=100, learning_rate=0.01, verbose=True):
        training_losses, validation_losses, training_accuracies, validation_accuracies = [], [], [], []
        self.learning_rate = learning_rate

        n_samples = X_train.shape[0]
        for epoch in range(epochs):
            total_loss, correct_predictions = 0, 0

            for x, y_true in zip(X_train, y_train):
                self.propagate_forward(x)
                loss = self.loss_f.function(y_true, self.a[-1])
                total_loss += loss

                if self.is_correct_prediction(self.a[-1], y_true):
                    correct_predictions += 1

                # Get gradients
                dW, db = self.propagate_backward(y_true)

                # Update weights and biases immediately
                for layer in range(1, len(self.W)):
                    self.W[layer] -= self.learning_rate * dW[layer]
                    self.b[layer] -= self.learning_rate * db[layer]

            # Calculate average loss and accuracy for the epoch
            avg_loss = total_loss / n_samples
            accuracy = (correct_predictions / n_samples) * 100
            training_losses.append(avg_loss)
            training_accuracies.append(accuracy)

            # Validation phase
            val_loss, val_correct_predictions = 0, 0
            for i in range(X_val.shape[0]):
                x, y_true = X_val[i], y_val[i]
                self.propagate_forward(x)
                val_loss += self.loss_f.function(y_true, self.a[-1])

                if self.is_correct_prediction(self.a[-1], y_true):
                    val_correct_predictions += 1

            val_avg_loss = val_loss / X_val.shape[0]
            val_accuracy = (val_correct_predictions / X_val.shape[0]) * 100
            validation_losses.append(val_avg_loss)
            validation_accuracies.append(val_accuracy)

            if verbose:
                print(f"Epoch {epoch+1}")
                print(f"Training Loss = {avg_loss:.4f}, Validation Loss = {val_avg_loss:.4f}")
                print(f"Training Accuracy: {accuracy:.2f}%, Validation Accuracy: {val_accuracy:.2f}%")
                print("----------------------------------------------------------------------")

        return training_losses, validation_losses, training_accuracies, validation_accuracies

    def train_mini_batch_gd(self, X_train, y_train, X_val, y_val, epochs=100, learning_rate=0.01, batch_size=32, verbose=True):
        training_losses, validation_losses, training_accuracies, validation_accuracies = [], [], [], []
        self.learning_rate = learning_rate
    
        n_samples = X_train.shape[0]
        for epoch in range(epochs):
            # Shuffle the data at the beginning of each epoch (optional but recommended)
            indices = np.arange(n_samples)
            np.random.shuffle(indices)
            X_train_shuffled = X_train[indices]
            y_train_shuffled = y_train[indices]
    
            total_loss, correct_predictions = 0, 0
    
            for start_idx in range(0, n_samples, batch_size):
                end_idx = min(start_idx + batch_size, n_samples)
                batch_X = X_train_shuffled[start_idx:end_idx]
                batch_y = y_train_shuffled[start_idx:end_idx]
    
                # Initialize batch gradients
                batch_dW = [None] + [np.zeros_like(w) for w in self.W[1:]]
                batch_db = [None] + [np.zeros_like(b) for b in self.b[1:]]
    
                batch_loss = 0
                batch_correct = 0
    
                for x, y_true in zip(batch_X, batch_y):
                    self.propagate_forward(x)
                    batch_loss += self.loss_f.function(y_true, self.a[-1])
    
                    if self.is_correct_prediction(self.a[-1], y_true):
                        batch_correct += 1
    
                    # Get gradients
                    dW, db = self.propagate_backward(y_true)
    
                    # Accumulate batch gradients
                    for layer in range(1, len(self.W)):
                        batch_dW[layer] += dW[layer]
                        batch_db[layer] += db[layer]
    
                # Compute average gradients for the mini-batch
                for layer in range(1, len(self.W)):
                    batch_dW[layer] /= (end_idx - start_idx)
                    batch_db[layer] /= (end_idx - start_idx)
    
                # Update weights and biases using mini-batch gradients
                for layer in range(1, len(self.W)):
                    self.W[layer] -= self.learning_rate * batch_dW[layer]
                    self.b[layer] -= self.learning_rate * batch_db[layer]
    
                # Accumulate loss and correct predictions
                total_loss += batch_loss
                correct_predictions += batch_correct
    
            # Average loss and accuracy for the epoch
            avg_loss = total_loss / n_samples
            accuracy = (correct_predictions / n_samples) * 100
            training_losses.append(avg_loss)
            training_accuracies.append(accuracy)
    
            # Validation phase
            val_loss, val_correct_predictions = 0, 0
            for i in range(X_val.shape[0]):
                x, y_true = X_val[i], y_val[i]
                self.propagate_forward(x)
                val_loss += self.loss_f.function(y_true, self.a[-1])
    
                if self.is_correct_prediction(self.a[-1], y_true):
                    val_correct_predictions += 1
    
            val_avg_loss = val_loss / X_val.shape[0]
            val_accuracy = (val_correct_predictions / X_val.shape[0]) * 100
            validation_losses.append(val_avg_loss)
            validation_accuracies.append(val_accuracy)
    
            if verbose:
                print(f"Epoch {epoch+1}")
                print(f"Training Loss = {avg_loss:.4f}, Validation Loss = {val_avg_loss:.4f}")
                print(f"Training Accuracy: {accuracy:.2f}%, Validation Accuracy: {val_accuracy:.2f}%")
                print("----------------------------------------------------------------------")
    
        return training_losses, validation_losses, training_accuracies, validation_accuracies

    def is_correct_prediction(self, predicted, true):
        if len(predicted) == 1:
            # Binary classification
            return ((predicted > 0.5).astype(int) == true).astype(int).sum()
        else:
            # Multiclass classification
            return np.argmax(predicted) == np.argmax(true)

    def predict(self, x):
        self.propagate_forward(x)
        y_pred = self.a[-1]
        return y_pred
    

epochs = 18

model = DenseNeuralNetwork(
    layer_sizes = [2, 16, 22, 5, 3], 
    activations = [None, ReLU, ReLU, ReLU, Softmax],  # Use Softmax in the output layer for multi-class probabilities
    loss_f = CategoricalCrossEntropy                  # Use cross-entropy loss
)

'''1) Mini-Batch GD WITH BATCH_SIZE=1'''
metrics = model.train_mini_batch_gd(X_train, y_train, X_val, y_val, epochs=epochs, batch_size=1, learning_rate=0.0001)

'''
Result:
Epoch 1
Training Loss = 5.6475, Validation Loss = 2.7214
Training Accuracy: 43.83%, Validation Accuracy: 42.00%
----------------------------------------------------------------------
Epoch 2
Training Loss = 1.3898, Validation Loss = 0.8878
Training Accuracy: 49.00%, Validation Accuracy: 62.50%
----------------------------------------------------------------------
Epoch 3
Training Loss = 0.8060, Validation Loss = 0.7497
Training Accuracy: 69.17%, Validation Accuracy: 72.00%
----------------------------------------------------------------------
Epoch 4
Training Loss = 0.7209, Validation Loss = 0.6795
Training Accuracy: 74.50%, Validation Accuracy: 76.00%
----------------------------------------------------------------------
...
'''

'''2) SGD'''
metrics = metrics = model.train_sgd(X_train, y_train, X_val, y_val, epochs=epochs, learning_rate=0.0001)

'''
Result:
Epoch 1
Training Loss = 5.8572, Validation Loss = 2.9794
Training Accuracy: 46.33%, Validation Accuracy: 44.50%
----------------------------------------------------------------------
Epoch 2
Training Loss = 1.4837, Validation Loss = 0.9170
Training Accuracy: 48.00%, Validation Accuracy: 62.00%
----------------------------------------------------------------------
Epoch 3
Training Loss = 0.8180, Validation Loss = 0.7587
Training Accuracy: 66.50%, Validation Accuracy: 72.00%
----------------------------------------------------------------------
Epoch 4
Training Loss = 0.7289, Validation Loss = 0.6890
Training Accuracy: 75.83%, Validation Accuracy: 74.50%
----------------------------------------------------------------------
...
'''