Unexpected NotFittedError for a fitted transformer passed to ColumnTransformer

### Describe the bug

Hi,

My ultimate goal is to use an already-trained classifier as a transformer in a new Scikit-learn Pipeline. The prediction of this model will be used as a feature in addition to other features (not used by the already-trained model). For this purpose, I created two custom transformers: `FeatureExtractor` and `PretrainedClassifierTransformer`. These two transformers are passed to the `ColumnTransformer`. However, when I call the ColumnTransformer's `fit()` method, I get the `NotFittedError` concerning the already-trained classifier, even though the classifier is already fitted.

### Steps/Code to Reproduce

```python
import numpy as np
import pandas as pd
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.compose import ColumnTransformer
from sklearn.datasets import make_classification
from sklearn.ensemble import RandomForestClassifier
from sklearn.utils.validation import check_is_fitted

def is_fitted(clf):
    return check_is_fitted(clf) is None

class FeatureExtractor(BaseEstimator, TransformerMixin):
    def __init__(self, features: list[str]):
        self.features = features

    def fit(self, X, y=None):
        return self

    def transform(self, X):
        return X[self.features].values
    
class PretrainedClassifierTransformer(BaseEstimator, TransformerMixin):
    def __init__(self, pretrained_clf, clf_inputs):
        self.pretrained_clf = pretrained_clf
        self.clf_inputs = clf_inputs
        self._is_fitted = True # is_fitted(pretrained_classifier)

    def fit(self, X, y=None):
        return self

    def transform(self, X):
        predictions = self.pretrained_clf.predict_proba(X[self.clf_inputs])
        return predictions[:, 1].reshape(-1, 1)
    
    def __sklearn_is_fitted__(self):
        """
        Check fitted status and return a Boolean value.
        """
        return hasattr(self, "_is_fitted") and self._is_fitted

X1, y1 = make_classification(n_samples=100, n_features=10, n_classes=1, random_state=1)
feature_names = [f"x{i:02d}" for i in range(X1.shape[1])]
X1 = pd.DataFrame(X1, columns=feature_names)

pretrained_clf = RandomForestClassifier()
pretrained_clf.fit(X1, y1)
print("Is `pretrained_clf` fitted?", is_fitted(pretrained_clf))

X2, y2 = make_classification(n_samples=100, n_features=10, n_classes=1, random_state=2)
X2 = pd.DataFrame(X2, columns=feature_names)

new_feature_names = ["x_new_1", "x_new_2"]
X2_new_features = pd.DataFrame(np.random.rand(X2.size, len(new_feature_names)), columns=new_feature_names)
X2_all = pd.concat([X2, X2_new_features], axis=1)

pre_trained_clf_tf = PretrainedClassifierTransformer(pretrained_clf, feature_names)
print("Is pre_trained_classifier_transformer fitted?", is_fitted(pre_trained_clf_tf))
print("Is pre_trained_classifier_tf.pretrained_classifier fitted?", is_fitted(pre_trained_clf_tf.pretrained_clf))

# pre_trained_clf_tf.fit(X2_all, y2)
column_transformer = ColumnTransformer(
    transformers=[
        ("feature_extractor", FeatureExtractor(new_feature_names), new_feature_names),
        ("pretrained_classifier", pre_trained_clf_tf, feature_names)
    ],
    remainder="drop",
)

column_transformer.fit(X2_all, y2)
```

### Expected Results

The expected result is that `column_transformer.fit(X2_all, y2)` works and generates no errors.

### Actual Results

```python
Is `pretrained_clf` fitted? True
Is pre_trained_classifier_transformer fitted? True
Is pre_trained_classifier_tf.pretrained_classifier fitted? True
---------------------------------------------------------------------------
NotFittedError                            Traceback (most recent call last)
/var/folders/3w/yzr41n292mn81j9sm82zs80r0000gn/T/ipykernel_51534/2673941823.py in <module>
     67 )
     68 
---> 69 column_transformer.fit(X2_all, y2)

~/Library/Caches/pypoetry/virtualenvs/algo-pnwHxCOU-py3.9/lib/python3.9/site-packages/sklearn/compose/_column_transformer.py in fit(self, X, y)
    715         # we use fit_transform to make sure to set sparse_output_ (for which we
    716         # need the transformed data) to have consistent output type in predict
--> 717         self.fit_transform(X, y=y)
    718         return self
    719 

~/Library/Caches/pypoetry/virtualenvs/algo-pnwHxCOU-py3.9/lib/python3.9/site-packages/sklearn/utils/_set_output.py in wrapped(self, X, *args, **kwargs)
    155     @wraps(f)
    156     def wrapped(self, X, *args, **kwargs):
--> 157         data_to_wrap = f(self, X, *args, **kwargs)
    158         if isinstance(data_to_wrap, tuple):
    159             # only wrap the first output for cross decomposition

~/Library/Caches/pypoetry/virtualenvs/algo-pnwHxCOU-py3.9/lib/python3.9/site-packages/sklearn/base.py in wrapper(estimator, *args, **kwargs)
   1150                 )
   1151             ):
-> 1152                 return fit_method(estimator, *args, **kwargs)
   1153 
   1154         return wrapper

~/Library/Caches/pypoetry/virtualenvs/algo-pnwHxCOU-py3.9/lib/python3.9/site-packages/sklearn/compose/_column_transformer.py in fit_transform(self, X, y)
    752         self._validate_remainder(X)
    753 
--> 754         result = self._fit_transform(X, y, _fit_transform_one)
    755 
    756         if not result:

~/Library/Caches/pypoetry/virtualenvs/algo-pnwHxCOU-py3.9/lib/python3.9/site-packages/sklearn/compose/_column_transformer.py in _fit_transform(self, X, y, func, fitted, column_as_strings)
    679         )
    680         try:
--> 681             return Parallel(n_jobs=self.n_jobs)(
    682                 delayed(func)(
    683                     transformer=clone(trans) if not fitted else trans,

~/Library/Caches/pypoetry/virtualenvs/algo-pnwHxCOU-py3.9/lib/python3.9/site-packages/sklearn/utils/parallel.py in __call__(self, iterable)
     63             for delayed_func, args, kwargs in iterable
     64         )
---> 65         return super().__call__(iterable_with_config)
     66 
     67 

~/Library/Caches/pypoetry/virtualenvs/algo-pnwHxCOU-py3.9/lib/python3.9/site-packages/joblib/parallel.py in __call__(self, iterable)
   1861             output = self._get_sequential_output(iterable)
   1862             next(output)
-> 1863             return output if self.return_generator else list(output)
   1864 
   1865         # Let's create an ID that uniquely identifies the current call. If the

~/Library/Caches/pypoetry/virtualenvs/algo-pnwHxCOU-py3.9/lib/python3.9/site-packages/joblib/parallel.py in _get_sequential_output(self, iterable)
   1790                 self.n_dispatched_batches += 1
   1791                 self.n_dispatched_tasks += 1
-> 1792                 res = func(*args, **kwargs)
   1793                 self.n_completed_tasks += 1
   1794                 self.print_progress()

~/Library/Caches/pypoetry/virtualenvs/algo-pnwHxCOU-py3.9/lib/python3.9/site-packages/sklearn/utils/parallel.py in __call__(self, *args, **kwargs)
    125             config = {}
    126         with config_context(**config):
--> 127             return self.function(*args, **kwargs)

~/Library/Caches/pypoetry/virtualenvs/algo-pnwHxCOU-py3.9/lib/python3.9/site-packages/sklearn/pipeline.py in _fit_transform_one(transformer, X, y, weight, message_clsname, message, **fit_params)
    955     with _print_elapsed_time(message_clsname, message):
    956         if hasattr(transformer, "fit_transform"):
--> 957             res = transformer.fit_transform(X, y, **fit_params)
    958         else:
    959             res = transformer.fit(X, y, **fit_params).transform(X)

~/Library/Caches/pypoetry/virtualenvs/algo-pnwHxCOU-py3.9/lib/python3.9/site-packages/sklearn/utils/_set_output.py in wrapped(self, X, *args, **kwargs)
    155     @wraps(f)
    156     def wrapped(self, X, *args, **kwargs):
--> 157         data_to_wrap = f(self, X, *args, **kwargs)
    158         if isinstance(data_to_wrap, tuple):
    159             # only wrap the first output for cross decomposition

~/Library/Caches/pypoetry/virtualenvs/algo-pnwHxCOU-py3.9/lib/python3.9/site-packages/sklearn/base.py in fit_transform(self, X, y, **fit_params)
    917         else:
    918             # fit method of arity 2 (supervised transformation)
--> 919             return self.fit(X, y, **fit_params).transform(X)
    920 
    921 

~/Library/Caches/pypoetry/virtualenvs/algo-pnwHxCOU-py3.9/lib/python3.9/site-packages/sklearn/utils/_set_output.py in wrapped(self, X, *args, **kwargs)
    155     @wraps(f)
    156     def wrapped(self, X, *args, **kwargs):
--> 157         data_to_wrap = f(self, X, *args, **kwargs)
    158         if isinstance(data_to_wrap, tuple):
    159             # only wrap the first output for cross decomposition

/var/folders/3w/yzr41n292mn81j9sm82zs80r0000gn/T/ipykernel_51534/2673941823.py in transform(self, X)
     30 
     31     def transform(self, X):
---> 32         predictions = self.pretrained_clf.predict_proba(X[self.clf_inputs])
     33         return predictions[:, 1].reshape(-1, 1)
     34 

~/Library/Caches/pypoetry/virtualenvs/algo-pnwHxCOU-py3.9/lib/python3.9/site-packages/sklearn/ensemble/_forest.py in predict_proba(self, X)
    861             classes corresponds to that in the attribute :term:`classes_`.
    862         """
--> 863         check_is_fitted(self)
    864         # Check data
    865         X = self._validate_X_predict(X)

~/Library/Caches/pypoetry/virtualenvs/algo-pnwHxCOU-py3.9/lib/python3.9/site-packages/sklearn/utils/validation.py in check_is_fitted(estimator, attributes, msg, all_or_any)
   1459 
   1460     if not _is_fitted(estimator, attributes, all_or_any):
-> 1461         raise NotFittedError(msg % {"name": type(estimator).__name__})
   1462 
   1463 

NotFittedError: This RandomForestClassifier instance is not fitted yet. Call 'fit' with appropriate arguments before using this estimator.
```

### Versions

```shell
System:
    python: 3.9.10 (main, Oct 27 2023, 17:01:40)  [Clang 15.0.0 (clang-1500.0.40.1)]
executable: /Users/jadd/Library/Caches/pypoetry/virtualenvs/algo-pnwHxCOU-py3.9/bin/python
   machine: macOS-13.5-arm64-arm-64bit

Python dependencies:
      sklearn: 1.3.2
          pip: 24.0
   setuptools: 65.7.0
        numpy: 1.21.4
        scipy: 1.7.3
       Cython: None
       pandas: 1.3.5
   matplotlib: 3.5.1
       joblib: 1.3.2
threadpoolctl: 3.0.0

Built with OpenMP: True

threadpoolctl info:
       user_api: openmp
   internal_api: openmp
         prefix: libomp
       filepath: /Users/jadd/Library/Caches/pypoetry/virtualenvs/algo-pnwHxCOU-py3.9/lib/python3.9/site-packages/sklearn/.dylibs/libomp.dylib
        version: None
    num_threads: 12

       user_api: blas
   internal_api: openblas
         prefix: libopenblas
       filepath: /Users/jadd/Library/Caches/pypoetry/virtualenvs/algo-pnwHxCOU-py3.9/lib/python3.9/site-packages/numpy/.dylibs/libopenblas.0.dylib
        version: 0.3.17
threading_layer: pthreads
   architecture: armv8
    num_threads: 12

       user_api: blas
   internal_api: openblas
         prefix: libopenblas
       filepath: /Users/jadd/Library/Caches/pypoetry/virtualenvs/algo-pnwHxCOU-py3.9/lib/python3.9/site-packages/scipy/.dylibs/libopenblas.0.dylib
        version: 0.3.17
threading_layer: pthreads
   architecture: armv8
    num_threads: 12
```

COMMENT:
That's because we still `clone` all estimators recursively, and that creates a new random forest in your case. If you want to control that behavior, you need to override `__sklearn_clone__` as bellow. Note that there's an indexing issue in your script, but unrelated to this issue:

```py
class PretrainedClassifierTransformer(BaseEstimator, TransformerMixin):
    def __init__(self, pretrained_clf, clf_inputs):
        self.pretrained_clf = pretrained_clf
        self.clf_inputs = clf_inputs
        self._is_fitted = True # is_fitted(pretrained_classifier)

    def fit(self, X, y=None):
        return self

    def transform(self, X):
        predictions = self.pretrained_clf.predict_proba(X[self.clf_inputs])
        return predictions[:, 1].reshape(-1, 1)
    
    def __sklearn_clone__(self):
        return PretrainedClassifierTransformer(self.pretrained_clf, self.clf_inputs)

    def __sklearn_is_fitted__(self):
        """
        Check fitted status and return a Boolean value.
        """
        return hasattr(self, "_is_fitted") and self._is_fitted
```

Labels: Bug, Needs Triage