GridSearchCV instant model convergence upon using many parameters

### Describe the bug

I am using `sklearn.model_selection.GridSearchCV` combined with a tensorflow/keras model wrapped using `scikeras.wrappers.KerasClassifier`. The image dataset used for classification can be downloaded [here](https://www.kaggle.com/datasets/puneet6060/intel-image-classification?select=seg_pred).

### Steps/Code to Reproduce

GridSearchCV implementation:
```
from sklearn.model_selection import GridSearchCV
from scikeras.wrappers import KerasClassifier

dataset = image_dataset_from_directory(
    './dataset',
    seed=123,
    image_size=(img_size, img_size)
)

images = np.concatenate([x for x, y in dataset], axis=0)
labels = np.concatenate([y for x, y in dataset], axis=0)

param_grid=dict(
    model__last_layer_param=[256, 512, 1024],
    model__augment=[False]
    # model__l2_param=[0.001, 0.0001],
    # model__last_dropout_param=[0.2, 0.3, 0.4]
)

model = KerasClassifier(model=create_model)

print(model.get_params())

grid = GridSearchCV(estimator=model, param_grid=param_grid)

grid_result = grid.fit(images, labels, epochs=20)
```

Original model which seems to instantly converge:
```
from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout, BatchNormalization
from tensorflow.keras.losses import SparseCategoricalCrossentropy
from tensorflow.keras.regularizers import l2

def create_model(l2_param=0.001, last_dropout_param=0.3, last_layer_param=512, augment=True):
    model = Sequential()

    if augment:
        model.add(data_augmentation)
        model.add(Rescaling(1./255))

    model.add(Conv2D(32, (3,3), input_shape= (img_size,img_size, 3), activation = 'relu', padding = 'same')) #padding = same size output
    model.add(MaxPooling2D())

    model.add(Conv2D(64, (3,3), activation = 'relu', padding = 'same'))
    model.add(MaxPooling2D())

    model.add(Conv2D(128, (3,3), activation = 'relu', padding = 'same')) 
    model.add(MaxPooling2D())

    model.add(Conv2D(256, (3,3), activation = 'relu', padding = 'same')) 
    model.add(MaxPooling2D())

    model.add(Conv2D(512, (3,3), activation = 'relu', padding = 'same', kernel_regularizer=l2(l=l2_param))) 
    model.add(MaxPooling2D())
    model.add(Dropout(0.1))

    model.add(Conv2D(1024, (3,3), activation = 'relu', padding = 'same', kernel_regularizer=l2(l=l2_param))) 
    model.add(MaxPooling2D())
    model.add(Dropout(0.1))

    if last_layer_param != 0:
        model.add(Conv2D(last_layer_param, (3,3), activation = 'relu', padding = 'same', kernel_regularizer=l2(l=l2_param))) 
        model.add(MaxPooling2D())

    model.add(Dropout(last_dropout_param))
    model.add(Flatten())

    model.add(Dense(256, activation='relu'))
    #model.add(Dense(512, activation='relu'))
    model.add(Dense(5, activation = 'softmax'))

    model.compile(
       optimizer = 'adam', 
       loss=SparseCategoricalCrossentropy(from_logits=False), 
       metrics=['accuracy']
    )

    return model
```

Adapted model (without last 3 layers) that shows some progress in accuracy:
```
from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout, BatchNormalization
from tensorflow.keras.losses import SparseCategoricalCrossentropy
from tensorflow.keras.regularizers import l2

def create_model(l2_param=0.001, last_dropout_param=0.3, last_layer_param=512, augment=True):
    model = Sequential()

    if augment:
        model.add(data_augmentation)
        model.add(Rescaling(1./255))

    model.add(Conv2D(32, (3,3), input_shape= (img_size,img_size, 3), activation = 'relu', padding = 'same')) #padding = same size output
    model.add(MaxPooling2D())

    model.add(Conv2D(64, (3,3), activation = 'relu', padding = 'same'))
    model.add(MaxPooling2D())

    model.add(Conv2D(128, (3,3), activation = 'relu', padding = 'same')) 
    model.add(MaxPooling2D())

    model.add(Conv2D(256, (3,3), activation = 'relu', padding = 'same')) 
    model.add(MaxPooling2D())

    model.add(Dropout(last_dropout_param))
    model.add(Flatten())

    model.add(Dense(256, activation='relu'))
    model.add(Dense(5, activation = 'softmax'))

    model.compile(
       optimizer = 'adam', 
       loss=SparseCategoricalCrossentropy(from_logits=False), 
       metrics=['accuracy']
    )

    return model
```

Data augmentation:
```
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import RandomRotation, RandomZoom, Rescaling, RandomFlip

data_augmentation = Sequential(
  [
    RandomFlip("horizontal", input_shape=(img_size, img_size, 3)),
    RandomRotation(0.1),
    RandomZoom(0.1),
  ]
)
```

### Expected Results

Upon using the full model with just normal training, I get the following results:
```
seed = 31
img_size = 150
train_ds = image_dataset_from_directory(
    './dataset/seg_train/seg_train',
    validation_split=0.2,
    subset="training",
    labels="inferred",
    seed=seed,
    image_size=(img_size, img_size),
    batch_size=128
)
val_ds = image_dataset_from_directory(
    './dataset/seg_train/seg_train',
    validation_split=0.2,
    subset="validation",
    labels="inferred",
    seed=seed,
    image_size=(img_size, img_size),
    batch_size=128
)
model = create_model(augment=True)
history = model.fit(train_ds, validation_data=val_ds, epochs=30)
```
![image](https://user-images.githubusercontent.com/31206249/224541928-9ed05718-632d-4799-abdb-6d458c2f504b.png)


I would expect GridSearchCV to show similar progress for the accuracies. 

### Actual Results

The below results are generated by using the provided GridSearchCV code.

#### Normal Model
The model seems to instantly converge when using GridSearchCV upon using the normal amount of layers and parameters in the model:
![image](https://user-images.githubusercontent.com/31206249/224541401-e316ead0-18fc-44cd-a841-582d07842b93.png)
model.summary():
```
Total params: 11,139,909
Trainable params: 11,139,909
Non-trainable params: 0
```
#### Adapted Model
While if I just remove the layers with the most parameters from the model, it shows at least some progress in the accuracy:
![Screenshot 2023-03-12 at 12 22 38](https://user-images.githubusercontent.com/31206249/224541350-51dfaf17-1f21-4e5d-b7a4-362a563e6b4b.png)
model.summary():
```
Total params: 5,698,373
Trainable params: 5,698,373
Non-trainable params: 0
```

### Versions

```shell
System:
    python: 3.10.9 | packaged by conda-forge | (main, Feb  2 2023, 20:26:08) [Clang 14.0.6 ]
executable: /opt/homebrew/Caskroom/miniforge/base/envs/.../bin/python
   machine: macOS-13.2.1-arm64-arm-64bit

Python dependencies:
      sklearn: 1.2.1
          pip: 23.0.1
   setuptools: 67.5.1
        numpy: 1.23.2
        scipy: 1.10.1
       Cython: None
       pandas: 1.5.3
   matplotlib: 3.7.1
       joblib: 1.2.0
threadpoolctl: 3.1.0

Built with OpenMP: True

threadpoolctl info:
       user_api: blas
   internal_api: openblas
         prefix: libopenblas
       filepath: /opt/homebrew/Caskroom/miniforge/base/envs/.../lib/libopenblas.0.dylib
        version: 0.3.21
threading_layer: openmp
   architecture: VORTEX
    num_threads: 10

       user_api: openmp
   internal_api: openmp
         prefix: libomp
       filepath: /opt/homebrew/Caskroom/miniforge/base/envs/.../lib/libomp.dylib
        version: None
    num_threads: 10
```

Tensorflow versions:
```
tensorflow-deps           2.10.0                        0    apple
tensorflow-macos          2.9.0                    pypi_0    pypi
tensorflow-metal          0.5.1                    pypi_0    pypi
```
```

COMMENT:
Never mind. The issue was on my side. The issue was that the images and labels did not correspond to each other correctly because the tensorflow works with batches. Here is the working code:
```
from sklearn.model_selection import GridSearchCV
from scikeras.wrappers import KerasClassifier
from tensorflow.keras.utils import image_dataset_from_directory
import numpy as np

img_size = 150

dataset = image_dataset_from_directory(
    './dataset',
    seed=123,  # for reproducibility
    image_size=(img_size, img_size)
)

ds = dataset.unbatch()
ds = list(ds.as_numpy_iterator())
images = np.array([x for x, y in ds])
labels = np.array([y for x, y in ds])

param_grid=dict(
    model__last_layer_param=[256, 512]
)

model = KerasClassifier(model=create_model)

print(model.get_params())

grid = GridSearchCV(estimator=model, param_grid=param_grid, cv=3)

grid_result = grid.fit(images, labels, epochs=10)
```

Labels: Bug, Needs Triage