jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def test_transform_snowpark_most_frequent_strategy_numeric(self) -> None:
        """
        Verify imputed data for "most frequent" strategy and numerical data.

        Raises
        ------
        AssertionError
            If the transformed result does not match the expected result.
        """
        input_cols = NUMERIC_COLS
        output_cols = OUTPUT_COLS
        df_pandas, df = framework_utils.get_df(self._session, DATA, SCHEMA)

        simple_imputer = SimpleImputer(input_cols=input_cols, output_cols=output_cols, strategy="most_frequent")
        simple_imputer.fit(df)

        df_none_nan_pandas, df_none_nan = framework_utils.get_df(self._session, DATA_NONE_NAN, SCHEMA)
        transformed_df = simple_imputer.transform(df_none_nan)

        simple_imputer_sklearn = SklearnSimpleImputer(strategy="most_frequent")
        simple_imputer_sklearn.fit(df_pandas[input_cols])
        sklearn_transformed_dataset = simple_imputer_sklearn.transform(df_none_nan_pandas[input_cols])

        np.testing.assert_allclose(transformed_df[output_cols].to_pandas().to_numpy(), sklearn_transformed_dataset)
//  [python]
def test_transform_sklearn_categorical(self) -> None:
        """
        Verify transform of pandas dataframe with categorical columns.

        Raises
        ------
        AssertionErrord
            If the transformed result does not match the one generated by Sklearn.
        """
        input_cols = CATEGORICAL_COLS
        output_cols = OUTPUT_COLS
        df_pandas, df = framework_utils.get_df(self._session, DATA, SCHEMA)

        simple_imputer = SimpleImputer(
            input_cols=input_cols,
            output_cols=output_cols,
            strategy="constant",
            fill_value="missing_value",
            missing_values=None,
        )
        simple_imputer.fit(df)

        df_none_nan_pandas, df_none_nan = framework_utils.get_df(self._session, DATA_NONE_NAN, SCHEMA)
        transformed_df = simple_imputer.transform(df_none_nan_pandas)

        sklearn_simple_imputer = Imputer(strategy="constant", fill_value="missing_value")
        sklearn_simple_imputer.fit(df_pandas[input_cols])
        sklearn_transformed_dataset = sklearn_simple_imputer.transform(df_none_nan_pandas[input_cols])

        np.testing.assert_equal(transformed_df[output_cols].to_numpy(), sklearn_transformed_dataset)
//  [python]
def test_transform_sklearn_categorical(self) -> None:
        """
        Verify transform of pandas dataframe with categorical columns.

        Raises
        ------
        AssertionErrord
            If the transformed result does not match the one generated by Sklearn.
        """
        input_cols = CATEGORICAL_COLS
        output_cols = OUTPUT_COLS
        df_pandas, df = framework_utils.get_df(self._session, DATA, SCHEMA)

        simple_imputer = SimpleImputer(
            input_cols=input_cols,
            output_cols=output_cols,
            strategy="constant",
            fill_value="missing_value",
            missing_values=None,
        )
        simple_imputer.fit(df)

        df_none_nan_pandas, df_none_nan = framework_utils.get_df(self._session, DATA_NONE_NAN, SCHEMA)
        transformed_df = simple_imputer.transform(df_none_nan_pandas)

        sklearn_simple_imputer = SklearnSimpleImputer(strategy="constant", missing_values=None)
        sklearn_simple_imputer.fit(df_pandas[input_cols])
        sklearn_transformed_dataset = sklearn_simple_imputer.transform(df_none_nan_pandas[input_cols])

        np.testing.assert_equal(transformed_df[output_cols].to_numpy(), sklearn_transformed_dataset)
//  [python]
def _init_preprocessor(self):
        self._label_encoder = LabelEncoder()
        # z-score
        self._scaler = StandardScaler()
        # remove nans
        self._imputer = Imputer(strategy="mean")
        # pca
        self._pca = PCA(n_components=self.n_components, whiten=True)
//  [python]
def _init_preprocessor(self):
        self._label_encoder = LabelEncoder()
        # z-score
        self._scaler = StandardScaler()
        # remove nans
        self._imputer = SimpleImputer(strategy="mean")
        # pca
        self._pca = PCA(n_components=self.n_components, whiten=True)
//  [python]
def test_imputation_most_frequent_pandas(dtype):
    # Test imputation using the most frequent strategy on pandas df
    pd = pytest.importorskip("pandas")

    f = io.StringIO(u"Cat1,Cat2,Cat3,Cat4\n"
                    ",i,x,\n"
                    "a,,y,\n"
                    "a,j,,\n"
                    "b,j,x,")

    df = pd.read_csv(f, dtype=dtype)

    X_true = np.array([
        ["a", "i", "x"],
        ["a", "j", "y"],
        ["a", "j", "x"],
        ["b", "j", "x"]
    ], dtype=object)

    imputer = Imputer(strategy="most_frequent")
    X_trans = imputer.fit_transform(df)

    assert_array_equal(X_trans, X_true)
//  [python]
def test_imputation_most_frequent_pandas(dtype):
    # Test imputation using the most frequent strategy on pandas df
    pd = pytest.importorskip("pandas")

    f = io.StringIO(u"Cat1,Cat2,Cat3,Cat4\n"
                    ",i,x,\n"
                    "a,,y,\n"
                    "a,j,,\n"
                    "b,j,x,")

    df = pd.read_csv(f, dtype=dtype)

    X_true = np.array([
        ["a", "i", "x"],
        ["a", "j", "y"],
        ["a", "j", "x"],
        ["b", "j", "x"]
    ], dtype=object)

    imputer = SimpleImputer(strategy="most_frequent")
    X_trans = imputer.fit_transform(df)

    assert_array_equal(X_trans, X_true)
//  [python]
def test_transform_sklearn_constant_string(self) -> None:
        """
        Verify imputed data using a string constant.

        Raises
        ------
        AssertionError
            If the transformed data does not match the expected result.
        """
        input_cols = CATEGORICAL_COLS
        output_cols = OUTPUT_COLS
        df_pandas, df = framework_utils.get_df(self._session, DATA, SCHEMA)

        fill_value = "foo"
        simple_imputer = SimpleImputer(
            input_cols=input_cols,
            output_cols=output_cols,
            strategy="constant",
            fill_value=fill_value,
            missing_values=None,
        )
        simple_imputer.fit(df)

        df_none_nan_pandas, df_none_nan = framework_utils.get_df(self._session, DATA_NONE_NAN, SCHEMA)
        transformed_df_sklearn = simple_imputer.transform(df_none_nan_pandas)

        sklearn_simple_imputer = Imputer(
            strategy="constant",
            fill_value=fill_value,
            missing_values=None,
        )
        sklearn_simple_imputer.fit(df_pandas[input_cols])
        sklearn_transformed_dataset = sklearn_simple_imputer.transform(df_none_nan_pandas[input_cols])

        np.testing.assert_equal(transformed_df_sklearn[output_cols].to_numpy(), sklearn_transformed_dataset)
//  [python]
def test_transform_sklearn_constant_string(self) -> None:
        """
        Verify imputed data using a string constant.

        Raises
        ------
        AssertionError
            If the transformed data does not match the expected result.
        """
        input_cols = CATEGORICAL_COLS
        output_cols = OUTPUT_COLS
        df_pandas, df = framework_utils.get_df(self._session, DATA, SCHEMA)

        fill_value = "foo"
        simple_imputer = SimpleImputer(
            input_cols=input_cols,
            output_cols=output_cols,
            strategy="constant",
            fill_value=fill_value,
            missing_values=None,
        )
        simple_imputer.fit(df)

        df_none_nan_pandas, df_none_nan = framework_utils.get_df(self._session, DATA_NONE_NAN, SCHEMA)
        transformed_df_sklearn = simple_imputer.transform(df_none_nan_pandas)

        sklearn_simple_imputer = SklearnSimpleImputer(
            strategy="constant",
            fill_value=fill_value,
            missing_values=None,
        )
        sklearn_simple_imputer.fit(df_pandas[input_cols])
        sklearn_transformed_dataset = sklearn_simple_imputer.transform(df_none_nan_pandas[input_cols])

        np.testing.assert_equal(transformed_df_sklearn[output_cols].to_numpy(), sklearn_transformed_dataset)
//  [python]
IMPUTERS = [IterativeImputer(), KNNImputer(), Imputer()]
SPARSE_IMPUTERS = [Imputer()]
//  [python]
IMPUTERS = [IterativeImputer(), KNNImputer(), SimpleImputer()]
SPARSE_IMPUTERS = [SimpleImputer()]
//  [python]
def test_imputation_constant_pandas(dtype):
    # Test imputation using the constant strategy on pandas df
    pd = pytest.importorskip("pandas")

    f = io.StringIO(u"Cat1,Cat2,Cat3,Cat4\n"
                    ",i,x,\n"
                    "a,,y,\n"
                    "a,j,,\n"
                    "b,j,x,")

    df = pd.read_csv(f, dtype=dtype)

    X_true = np.array([
        ["missing_value", "i", "x", "missing_value"],
        ["a", "missing_value", "y", "missing_value"],
        ["a", "j", "missing_value", "missing_value"],
        ["b", "j", "x", "missing_value"]
    ], dtype=object)

    imputer = Imputer(strategy="constant")
    X_trans = imputer.fit_transform(df)

    assert_array_equal(X_trans, X_true)
//  [python]
def test_imputation_constant_pandas(dtype):
    # Test imputation using the constant strategy on pandas df
    pd = pytest.importorskip("pandas")

    f = io.StringIO(u"Cat1,Cat2,Cat3,Cat4\n"
                    ",i,x,\n"
                    "a,,y,\n"
                    "a,j,,\n"
                    "b,j,x,")

    df = pd.read_csv(f, dtype=dtype)

    X_true = np.array([
        ["missing_value", "i", "x", "missing_value"],
        ["a", "missing_value", "y", "missing_value"],
        ["a", "j", "missing_value", "missing_value"],
        ["b", "j", "x", "missing_value"]
    ], dtype=object)

    imputer = SimpleImputer(strategy="constant")
    X_trans = imputer.fit_transform(df)

    assert_array_equal(X_trans, X_true)
//  [python]
def naive_encoder(numerical_features, cat_features) -> TransformerMixin:
    """Create a naive encoder for categorical and numerical features.

    The encoder handles nans for all features and uses label encoder for categorical features. Then, all features are
    scaled using RobustScaler.

    Parameters
    ----------
    numerical_features
    cat_features

    Returns
    -------
    TransformerMixin
        A transformer object
    """
    return ColumnTransformer(
        transformers=[
            ('num', Pipeline([
                ('nan_handling', Imputer()),
                ('norm', RobustScaler())
            ]),
                np.array(numerical_features, dtype='object')),
            ('cat',
             Pipeline([
                 ('nan_handling', Imputer(strategy='most_frequent')),
                 ('encode', run_available_kwargs(OrdinalEncoder, handle_unknown='use_encoded_value', unknown_value=-1)),
                 ('norm', RobustScaler())
             ]),
             np.array(cat_features, dtype='object'))
        ]
    )
//  [python]
def naive_encoder(numerical_features, cat_features) -> TransformerMixin:
    """Create a naive encoder for categorical and numerical features.

    The encoder handles nans for all features and uses label encoder for categorical features. Then, all features are
    scaled using RobustScaler.

    Parameters
    ----------
    numerical_features
    cat_features

    Returns
    -------
    TransformerMixin
        A transformer object
    """
    return ColumnTransformer(
        transformers=[
            ('num', Pipeline([
                ('nan_handling', SimpleImputer()),
                ('norm', RobustScaler())
            ]),
                np.array(numerical_features, dtype='object')),
            ('cat',
             Pipeline([
                 ('nan_handling', SimpleImputer(strategy='most_frequent')),
                 ('encode', run_available_kwargs(OrdinalEncoder, handle_unknown='use_encoded_value', unknown_value=-1)),
                 ('norm', RobustScaler())
             ]),
             np.array(cat_features, dtype='object'))
        ]
    )
//  [python]
def fit(self, X, y=None, categorical_feature=None, numerical_feature=None, **kwargs):
        X = super(SimpleImputer, self).fit(X, y, categorical_feature, numerical_feature)
        cat_cols = self.categorical_feature
        num_cols = self.numerical_feature
        if num_cols.size:
            self.num_imputer = Imputer(strategy=self.num_strategy).fit(X[num_cols])
        else:
            self.num_imputer = None
        if cat_cols.size:
            self.cat_imputer = CategoricalImputer(strategy=self.cat_strategy).fit(X[cat_cols])
        else:
            self.cat_imputer = None
        self.cat_cols = cat_cols
        self.num_cols = num_cols
        return self
//  [python]
def fit(self, X, y=None, categorical_feature=None, numerical_feature=None, **kwargs):
        X = super(SimpleImputer, self).fit(X, y, categorical_feature, numerical_feature)
        cat_cols = self.categorical_feature
        num_cols = self.numerical_feature
        if num_cols.size:
            self.num_imputer = SklearnSimpleImputer(strategy=self.num_strategy).fit(X[num_cols])
        else:
            self.num_imputer = None
        if cat_cols.size:
            self.cat_imputer = CategoricalImputer(strategy=self.cat_strategy).fit(X[cat_cols])
        else:
            self.cat_imputer = None
        self.cat_cols = cat_cols
        self.num_cols = num_cols
        return self
//  [python]
validation_dataset['quality'] = (validation_data['quality'] == "Good").astype(int)

# Gather the numerical and categorical features
numerical_features = X_training._get_numeric_data().columns.tolist()
categorical_features = list(set(X_training.columns) - set(numerical_features))

preprocessing_pipeline = []

one_hot_pipeline = Pipeline(steps=[
    ("imputer", Imputer(missing_values=None, strategy="constant", fill_value="")),
    ("onehot", OneHotEncoder(handle_unknown="ignore"))
])
//  [python]
validation_dataset['quality'] = (validation_data['quality'] == "Good").astype(int)

# Gather the numerical and categorical features
numerical_features = X_training._get_numeric_data().columns.tolist()
categorical_features = list(set(X_training.columns) - set(numerical_features))

preprocessing_pipeline = []

one_hot_pipeline = Pipeline(steps=[
    ("imputer", SimpleImputer(missing_values=None, strategy="constant", fill_value="")),
    ("onehot", OneHotEncoder(handle_unknown="ignore"))
])
//  [python]
def test_within_pipeline(self):
        """
        Test that visualizer can be accessed within a sklearn pipeline
        """
        X, y = load_spam()

        model = Pipeline([
            ('imputer', Imputer(missing_values=np.nan, strategy='mean')),
            ('dt', DiscriminationThreshold(LogisticRegression(multi_class="auto", solver="liblinear"), random_state=42))
        ])

        model.fit(X, y)
        model['dt'].finalize()
        self.assert_images_similar(model['dt'], tol=15)
//  [python]
def test_within_pipeline(self):
        """
        Test that visualizer can be accessed within a sklearn pipeline
        """
        X, y = load_spam()

        model = Pipeline([
            ('imputer', SimpleImputer(missing_values=np.nan, strategy='mean')),
            ('dt', DiscriminationThreshold(LogisticRegression(multi_class="auto", solver="liblinear"), random_state=42))
        ])

        model.fit(X, y)
        model['dt'].finalize()
        self.assert_images_similar(model['dt'], tol=15)
//  [python]
def test_within_pipeline_quickmethod(self):
        """
        Test that visualizer quickmethod can be accessed within a
        sklearn pipeline
        """
        X, y = load_spam()

        model = Pipeline([
            ('imputer', Imputer(missing_values=np.nan, strategy='mean')),
            ('dt',
             discrimination_threshold(LogisticRegression(multi_class="auto", solver="liblinear"), X,
                                      y, random_state=42))
        ])
        model['dt'].finalize()
        self.assert_images_similar(model['dt'], tol=15)
//  [python]
def test_within_pipeline_quickmethod(self):
        """
        Test that visualizer quickmethod can be accessed within a
        sklearn pipeline
        """
        X, y = load_spam()

        model = Pipeline([
            ('imputer', SimpleImputer(missing_values=np.nan, strategy='mean')),
            ('dt',
             discrimination_threshold(LogisticRegression(multi_class="auto", solver="liblinear"), X,
                                      y, random_state=42))
        ])
        model['dt'].finalize()
        self.assert_images_similar(model['dt'], tol=15)
//  [python]
def test_pipeline_as_model_input(self):
        """
        Test that visualizer can handle sklearn pipeline as model input
        """
        X, y = load_spam()

        model = Pipeline([
            ('imputer', Imputer(missing_values=np.nan, strategy='mean')),
            ('lr', LogisticRegression(multi_class="auto", solver="liblinear"))
        ])

        oz = DiscriminationThreshold(model, random_state=42)
        oz.fit(X, y)
        oz.finalize()
        self.assert_images_similar(oz, tol=15)
//  [python]
def test_pipeline_as_model_input(self):
        """
        Test that visualizer can handle sklearn pipeline as model input
        """
        X, y = load_spam()

        model = Pipeline([
            ('imputer', SimpleImputer(missing_values=np.nan, strategy='mean')),
            ('lr', LogisticRegression(multi_class="auto", solver="liblinear"))
        ])

        oz = DiscriminationThreshold(model, random_state=42)
        oz.fit(X, y)
        oz.finalize()
        self.assert_images_similar(oz, tol=15)
//  [python]
def test_pipeline(self):
        columns = [
            "pclass",
            "name",
            "sex",
            "age",
            "sibsp",
            "parch",
            "ticket",
            "fare",
            "cabin",
            "embarked",
            "boat",
            "body",
            "home.dest",
        ]

        numeric_features = ["age", "fare"]
        numeric_transformer = Pipeline(
            steps=[
                ("imputer", Imputer(strategy="median")),
                ("scaler", StandardScaler()),
            ]
        )

        categorical_features = ["embarked", "sex", "pclass"]
        categorical_transformer = Pipeline(
            steps=[
                ("imputer", Imputer(strategy="constant", fill_value="missing")),
                ("onehot", OneHotEncoder(handle_unknown="ignore")),
            ]
        )

        preprocessor = ColumnTransformer(
            transformers=[
                ("num", numeric_transformer, numeric_features),
                ("cat", categorical_transformer, categorical_features),
            ]
        )

        clf = Pipeline(
            steps=[
                ("preprocessor", preprocessor),
                ("classifier", LogisticRegression(solver="lbfgs")),
            ]
        )
        dot = pipeline2dot(clf, columns)
        self.assertIn("digraph{", dot)
        self.assertIn("StandardScaler", dot)
//  [python]
def test_pipeline(self):
        columns = [
            "pclass",
            "name",
            "sex",
            "age",
            "sibsp",
            "parch",
            "ticket",
            "fare",
            "cabin",
            "embarked",
            "boat",
            "body",
            "home.dest",
        ]

        numeric_features = ["age", "fare"]
        numeric_transformer = Pipeline(
            steps=[
                ("imputer", SimpleImputer(strategy="median")),
                ("scaler", StandardScaler()),
            ]
        )

        categorical_features = ["embarked", "sex", "pclass"]
        categorical_transformer = Pipeline(
            steps=[
                ("imputer", SimpleImputer(strategy="constant", fill_value="missing")),
                ("onehot", OneHotEncoder(handle_unknown="ignore")),
            ]
        )

        preprocessor = ColumnTransformer(
            transformers=[
                ("num", numeric_transformer, numeric_features),
                ("cat", categorical_transformer, categorical_features),
            ]
        )

        clf = Pipeline(
            steps=[
                ("preprocessor", preprocessor),
                ("classifier", LogisticRegression(solver="lbfgs")),
            ]
        )
        dot = pipeline2dot(clf, columns)
        self.assertIn("digraph{", dot)
        self.assertIn("StandardScaler", dot)
//  [python]
def test_pipeline_as_model_input_quickmethod(self):
        """
        Test that visualizer can handle sklearn pipeline as model input
        within a quickmethod
        """
        X, y = load_spam()

        model = Pipeline([
            ('imputer', Imputer(missing_values=np.nan, strategy='mean')),
            ('lr', LogisticRegression(multi_class="auto", solver="liblinear"))
        ])

        oz = discrimination_threshold(model, X, y, random_state=42)
        self.assert_images_similar(oz, tol=15)
//  [python]
def test_pipeline_as_model_input_quickmethod(self):
        """
        Test that visualizer can handle sklearn pipeline as model input
        within a quickmethod
        """
        X, y = load_spam()

        model = Pipeline([
            ('imputer', SimpleImputer(missing_values=np.nan, strategy='mean')),
            ('lr', LogisticRegression(multi_class="auto", solver="liblinear"))
        ])

        oz = discrimination_threshold(model, X, y, random_state=42)
        self.assert_images_similar(oz, tol=15)
//  [python]
def handle_nans(train_data: pd.DataFrame,
                val_data: Optional[pd.DataFrame] = None,
                constant: Optional[int] = 0,
                identifier: Optional[str] = "District") -> Union[pd.DataFrame, Tuple[pd.DataFrame, pd.DataFrame]]:
    """Imputes missing values in a dataframe. Currently only constant imputation is supported."""
    if val_data is not None:
        assert list(train_data.columns) == list(val_data.columns)
        val_data = val_data.copy()
    train_data = train_data.copy()

    columns = list(train_data.columns)
    if identifier in columns:
        columns.remove(identifier)

    imp = Imputer(missing_values=np.nan, strategy="constant", fill_value=constant)

    for col in columns:
        train_nans = get_nans(train_data[[col]])
        if train_nans.values > 0:
            tmp_imp = copy.deepcopy(imp)
            tmp_imp.fit(train_data[[col]])
            train_data[col] = tmp_imp.transform(train_data[[col]])
        if val_data is not None:
            val_nans = get_nans(train_nans[[col]])
            if val_nans.values > 0:
                tmp_imp = copy.deepcopy(imp)
                tmp_imp = copy.deepcopy(tmp_imp)
                tmp_imp.fit(val_data[[col]])
                val_data[col] = tmp_imp.transform(val_data[[col]])

    if val_data is not None:
        return train_data, val_data
    return train_data
//  [python]
def handle_nans(train_data: pd.DataFrame,
                val_data: Optional[pd.DataFrame] = None,
                constant: Optional[int] = 0,
                identifier: Optional[str] = "District") -> Union[pd.DataFrame, Tuple[pd.DataFrame, pd.DataFrame]]:
    """Imputes missing values in a dataframe. Currently only constant imputation is supported."""
    if val_data is not None:
        assert list(train_data.columns) == list(val_data.columns)
        val_data = val_data.copy()
    train_data = train_data.copy()

    columns = list(train_data.columns)
    if identifier in columns:
        columns.remove(identifier)

    imp = SimpleImputer(missing_values=np.nan, strategy="constant", fill_value=constant)

    for col in columns:
        train_nans = get_nans(train_data[[col]])
        if train_nans.values > 0:
            tmp_imp = copy.deepcopy(imp)
            tmp_imp.fit(train_data[[col]])
            train_data[col] = tmp_imp.transform(train_data[[col]])
        if val_data is not None:
            val_nans = get_nans(train_nans[[col]])
            if val_nans.values > 0:
                tmp_imp = copy.deepcopy(imp)
                tmp_imp = copy.deepcopy(tmp_imp)
                tmp_imp.fit(val_data[[col]])
                val_data[col] = tmp_imp.transform(val_data[[col]])

    if val_data is not None:
        return train_data, val_data
    return train_data
//  [python]
def create_preprocessor(
    impute_strategy, max_category_levels, unique_category_str, continuous_features, skewed_features, onehot_features, embed_features, bool_features
):
    """Creates sklearn ColumnTransformer that can be fit to training data to preprocess it for tabular neural network."""
    transformers = []  # order of various column transformers in this list is important!
    if continuous_features:
        continuous_transformer = Pipeline(steps=[("imputer", Imputer(strategy=impute_strategy)), ("scaler", StandardScaler())])
        transformers.append(("continuous", continuous_transformer, continuous_features))
    if skewed_features:
        power_transformer = Pipeline(
            steps=[("imputer", Imputer(strategy=impute_strategy)), ("quantile", QuantileTransformer(output_distribution="normal"))]
        )  # Or output_distribution = 'uniform'
        transformers.append(("skewed", power_transformer, skewed_features))
    if onehot_features:
        onehot_transformer = Pipeline(
            steps=[("onehot", OneHotMergeRaresHandleUnknownEncoder(max_levels=max_category_levels, sparse=False))]
        )  # test-time unknown values will be encoded as all zeros vector
        transformers.append(("onehot", onehot_transformer, onehot_features))
    if embed_features:  # Ordinal transformer applied to convert to-be-embedded categorical features to integer levels
        ordinal_transformer = Pipeline(
            steps=[("ordinal", OrdinalMergeRaresHandleUnknownEncoder(max_levels=max_category_levels))]
        )  # returns 0-n when max_category_levels = n-1. category n is reserved for unknown test-time categories.
        transformers.append(("ordinal", ordinal_transformer, embed_features))
    return ColumnTransformer(
        transformers=transformers, remainder="passthrough"
    )
//  [python]
def create_preprocessor(
    impute_strategy, max_category_levels, unique_category_str, continuous_features, skewed_features, onehot_features, embed_features, bool_features
):
    """Creates sklearn ColumnTransformer that can be fit to training data to preprocess it for tabular neural network."""
    transformers = []  # order of various column transformers in this list is important!
    if continuous_features:
        continuous_transformer = Pipeline(steps=[("imputer", SimpleImputer(strategy=impute_strategy)), ("scaler", StandardScaler())])
        transformers.append(("continuous", continuous_transformer, continuous_features))
    if skewed_features:
        power_transformer = Pipeline(
            steps=[("imputer", SimpleImputer(strategy=impute_strategy)), ("quantile", QuantileTransformer(output_distribution="normal"))]
        )  # Or output_distribution = 'uniform'
        transformers.append(("skewed", power_transformer, skewed_features))
    if onehot_features:
        onehot_transformer = Pipeline(
            steps=[("onehot", OneHotMergeRaresHandleUnknownEncoder(max_levels=max_category_levels, sparse=False))]
        )  # test-time unknown values will be encoded as all zeros vector
        transformers.append(("onehot", onehot_transformer, onehot_features))
    if embed_features:  # Ordinal transformer applied to convert to-be-embedded categorical features to integer levels
        ordinal_transformer = Pipeline(
            steps=[("ordinal", OrdinalMergeRaresHandleUnknownEncoder(max_levels=max_category_levels))]
        )  # returns 0-n when max_category_levels = n-1. category n is reserved for unknown test-time categories.
        transformers.append(("ordinal", ordinal_transformer, embed_features))
    return ColumnTransformer(
        transformers=transformers, remainder="passthrough"
    )
//  [python]
def simple_proc_for_tree_algoritms(numeric_features, categoric_features):
    """
    Create a simple preprocessing pipeline for tree based algorithms
    """

    catpipe = make_pipeline(
        ColumnsSelector(categoric_features),
        OrdinalEncoder(min_support=5)
        # ColumnApplier(FillNaN('nan')),
        # ColumnApplier(TolerantLabelEncoder())
    )
    numpipe = make_pipeline(
        ColumnsSelector(numeric_features),
        Imputer(strategy='mean'),
        StandardScaler(),
    )
    if numeric_features and categoric_features:
        return make_union(catpipe, numpipe)
    elif numeric_features:
        return numpipe
    elif categoric_features:
        return catpipe
    raise Exception('Both variable lists are empty')
//  [python]
def simple_proc_for_tree_algoritms(numeric_features, categoric_features):
    """
    Create a simple preprocessing pipeline for tree based algorithms
    """

    catpipe = make_pipeline(
        ColumnsSelector(categoric_features),
        OrdinalEncoder(min_support=5)
        # ColumnApplier(FillNaN('nan')),
        # ColumnApplier(TolerantLabelEncoder())
    )
    numpipe = make_pipeline(
        ColumnsSelector(numeric_features),
        SimpleImputer(strategy='mean'),
        StandardScaler(),
    )
    if numeric_features and categoric_features:
        return make_union(catpipe, numpipe)
    elif numeric_features:
        return numpipe
    elif categoric_features:
        return catpipe
    raise Exception('Both variable lists are empty')
//  [python]
def _make_cont_enc(df, conts) -> SKLearnEncoder:
    cont_enc = make_pipeline(
        StandardScaler(),
        Imputer(fill_value=0)
    ).fit(df[conts].values)
    return cont_enc
//  [python]
def _make_cont_enc(df, conts) -> SKLearnEncoder:
    cont_enc = make_pipeline(
        StandardScaler(),
        SimpleImputer(fill_value=0)
    ).fit(df[conts].values)
    return cont_enc
//  [python]
X = train.drop(["SalePrice", "Id"], axis=1)
y = np.log(train["SalePrice"])  # Log transformation
test_ids = test["Id"]
test = test.drop(["Id"], axis=1)

# Preprocessing for numerical data
numerical_transformer = Pipeline(
    steps=[("imputer", Imputer(strategy="median")), ("scaler", StandardScaler())]
)
//  [python]
X = train.drop(["SalePrice", "Id"], axis=1)
y = np.log(train["SalePrice"])  # Log transformation
test_ids = test["Id"]
test = test.drop(["Id"], axis=1)

# Preprocessing for numerical data
numerical_transformer = Pipeline(
    steps=[("imputer", SimpleImputer(strategy="median")), ("scaler", StandardScaler())]
)
//  [python]
def simple_proc_for_linear_algoritms(numeric_features, categoric_features):
    """
    Create a simple preprocessing pipeline for linear algorithms
    """

    catpipe = make_pipeline(
        ColumnsSelector(categoric_features),
        AsString(),
        ce.OneHotEncoder()
        # ColumnApplier(FillNaN('nan')),
        # ColumnApplier(TolerantLabelEncoder())
    )
    numpipe = make_pipeline(
        ColumnsSelector(numeric_features),
        Imputer(strategy='mean'),
        StandardScaler(),
    )
    if numeric_features and categoric_features:
        return make_union(catpipe, numpipe)
    elif numeric_features:
        return numpipe
    elif categoric_features:
        return catpipe
    raise Exception('Both variable lists are empty')
//  [python]
def simple_proc_for_linear_algoritms(numeric_features, categoric_features):
    """
    Create a simple preprocessing pipeline for linear algorithms
    """

    catpipe = make_pipeline(
        ColumnsSelector(categoric_features),
        AsString(),
        ce.OneHotEncoder()
        # ColumnApplier(FillNaN('nan')),
        # ColumnApplier(TolerantLabelEncoder())
    )
    numpipe = make_pipeline(
        ColumnsSelector(numeric_features),
        SimpleImputer(strategy='mean'),
        StandardScaler(),
    )
    if numeric_features and categoric_features:
        return make_union(catpipe, numpipe)
    elif numeric_features:
        return numpipe
    elif categoric_features:
        return catpipe
    raise Exception('Both variable lists are empty')
//  [python]
def read_plink_genotypes(self, bed):
        X = bed.compute().astype(np.float64)
        if np.any(np.isnan(X)):
            imp = Imputer(missing_values=np.nan, strategy='mean', copy=False)
            imp.fit(X)
            X = imp.transform(X)
        X -= X.mean(axis=0)
        assert not np.any(np.isnan(X))
        is_polymorphic = X.std(axis=0)>0
        X[:, is_polymorphic] /= X[:, is_polymorphic].std(axis=0)
        return X
//  [python]
def read_plink_genotypes(self, bed):
        X = bed.compute().astype(np.float64)
        if np.any(np.isnan(X)):
            imp = SimpleImputer(missing_values=np.nan, strategy='mean', copy=False)
            imp.fit(X)
            X = imp.transform(X)
        X -= X.mean(axis=0)
        assert not np.any(np.isnan(X))
        is_polymorphic = X.std(axis=0)>0
        X[:, is_polymorphic] /= X[:, is_polymorphic].std(axis=0)
        return X
//  [python]
intv_imputer = Imputer(strategy='median')
intv_imputer = intv_imputer.fit(dm_traindf[dm_interval_input])
fullX_intv_imp = intv_imputer.transform(dm_inputdf[dm_interval_input])

# Impute class missing values to most_frequent
class_imputer = Imputer(strategy='most_frequent')
//  [python]
intv_imputer = SimpleImputer(strategy='median')
intv_imputer = intv_imputer.fit(dm_traindf[dm_interval_input])
fullX_intv_imp = intv_imputer.transform(dm_inputdf[dm_interval_input])

# Impute class missing values to most_frequent
class_imputer = SimpleImputer(strategy='most_frequent')
//  [python]
def encode_features(features):
    '''特征 one-hot 编码'''

    logging.info('encoding features ...')

    # 为了让编码器正常工作，先补全缺失特征
    encoder = OneHotEncoder(
        sparse_output=True,
        dtype=numpy.float32,
        handle_unknown='ignore'
    )
    features = encoder.fit(
        Imputer(
            missing_values='',
            strategy='most_frequent'
        ).fit_transform(features)
    ).transform(features)

    categories = numpy.asarray([len(c) for c in encoder.categories_])

    logging.info('done. totally {} features'.format(features.shape[1]))
    return features, categories
//  [python]
def encode_features(features):
    '''特征 one-hot 编码'''

    logging.info('encoding features ...')

    # 为了让编码器正常工作，先补全缺失特征
    encoder = OneHotEncoder(
        sparse_output=True,
        dtype=numpy.float32,
        handle_unknown='ignore'
    )
    features = encoder.fit(
        SimpleImputer(
            missing_values='',
            strategy='most_frequent'
        ).fit_transform(features)
    ).transform(features)

    categories = numpy.asarray([len(c) for c in encoder.categories_])

    logging.info('done. totally {} features'.format(features.shape[1]))
    return features, categories
//  [python]
def main() -> None:
    # 处理缺失值
    imputer = Imputer(missing_values=np.nan, strategy='mean')
    data = pd.DataFrame(imputer.fit_transform(pd.DataFrame(get_font_data())))
    # 取出特征值\目标值
    x = data.drop([0], axis=1)
    y = data[0]
    # 分割数据集
    # x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=0)
    x_train = x.head(30)
    y_train = y.head(30)
    x_test = x.tail(10)
    y_test = y.tail(10)
    # 标准化
    # std = StandardScaler()
    # x_train = std.fit_transform(x_train)
    # x_test = std.transform(x_test)
    # 进行算法流程
    knn = KNeighborsClassifier(n_neighbors=1)
    # 开始训练
    knn.fit(x_train, y_train)
    # 预测结果
    y_predict = knn.predict(x_test)
    print(y)
    # 得出准确率
    print(knn.score(x_test, y_test))
//  [python]
def main() -> None:
    # 处理缺失值
    imputer = SimpleImputer(missing_values=np.nan, strategy='mean')
    data = pd.DataFrame(imputer.fit_transform(pd.DataFrame(get_font_data())))
    # 取出特征值\目标值
    x = data.drop([0], axis=1)
    y = data[0]
    # 分割数据集
    # x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=0)
    x_train = x.head(30)
    y_train = y.head(30)
    x_test = x.tail(10)
    y_test = y.tail(10)
    # 标准化
    # std = StandardScaler()
    # x_train = std.fit_transform(x_train)
    # x_test = std.transform(x_test)
    # 进行算法流程
    knn = KNeighborsClassifier(n_neighbors=1)
    # 开始训练
    knn.fit(x_train, y_train)
    # 预测结果
    y_predict = knn.predict(x_test)
    print(y)
    # 得出准确率
    print(knn.score(x_test, y_test))
//  [python]
def fit(self, X: pd.DataFrame, y=None):
        super().fit(X)

        # Real-valued features imputation
        feat_list = [f for f in self._real_feat if f in X]
        self._transf['real'] = Imputer(strategy=self.strategy).fit(X[feat_list].astype(np.float32))

        return self
//  [python]
def fit(self, X: pd.DataFrame, y=None):
        super().fit(X)

        # Real-valued features imputation
        feat_list = [f for f in self._real_feat if f in X]
        self._transf['real'] = SimpleImputer(strategy=self.strategy).fit(X[feat_list].astype(np.float32))

        return self
//  [python]
def pipeline():
    data_dir = Path("../data/titanic")
    # load data
    df_train = pd.read_csv(data_dir / "train.csv")
    df_test = pd.read_csv(data_dir / "test.csv")

    # remove row with many missing values
    df_train.drop(columns=["Cabin"])

    # split data
    df_train0, df_val = train_test_split(df_train, test_size=0.1)
    X_train = df_train0.copy()
    y_train = X_train.pop("Survived")

    X_val = df_val.copy()
    y_val = X_val.pop("Survived")

    # feature engineering for categorical features
    cat_cols = ["Embarked", "Sex", "Pclass"]
    cat_transformer = Pipeline(
        steps=[
            ("imputer", Imputer(strategy="most_frequent")),
            ("onehot", OneHotEncoder(handle_unknown="ignore", sparse=False)),
        ]
    )

    # feature engineering for numeric features
    num_cols = ["Age", "Fare"]
    num_transformer = Pipeline(
        steps=[("imputer", KNNImputer(n_neighbors=5)), ("scaler", RobustScaler())]
    )

    # Full data engineering
    preprocessor = ColumnTransformer(
        transformers=[
            ("num", num_transformer, num_cols),
            ("cat", cat_transformer, cat_cols),
        ]
    )

    # Full training pipeline
    clf = Pipeline(
        steps=[("preprocessor", preprocessor), ("classifier", RandomForestClassifier())]
    )

    # training
    clf.fit(X_train, y_train)

    # validation
    y_pred = clf.predict(X_val)
    print(f"Accuracy score {accuracy_score(list(y_val), list(y_pred))}")

    df_test.drop(columns=["Cabin"])
    preds = clf.predict(df_test)
    sample_submission = pd.read_csv(data_dir / "gender_submission.csv")
    sample_submission["Survived"] = preds
    sample_submission.to_csv("titanic_submission.csv", index=False)
//  [python]
def pipeline():
    data_dir = Path("../data/titanic")
    # load data
    df_train = pd.read_csv(data_dir / "train.csv")
    df_test = pd.read_csv(data_dir / "test.csv")

    # remove row with many missing values
    df_train.drop(columns=["Cabin"])

    # split data
    df_train0, df_val = train_test_split(df_train, test_size=0.1)
    X_train = df_train0.copy()
    y_train = X_train.pop("Survived")

    X_val = df_val.copy()
    y_val = X_val.pop("Survived")

    # feature engineering for categorical features
    cat_cols = ["Embarked", "Sex", "Pclass"]
    cat_transformer = Pipeline(
        steps=[
            ("imputer", SimpleImputer(strategy="most_frequent")),
            ("onehot", OneHotEncoder(handle_unknown="ignore", sparse=False)),
        ]
    )

    # feature engineering for numeric features
    num_cols = ["Age", "Fare"]
    num_transformer = Pipeline(
        steps=[("imputer", KNNImputer(n_neighbors=5)), ("scaler", RobustScaler())]
    )

    # Full data engineering
    preprocessor = ColumnTransformer(
        transformers=[
            ("num", num_transformer, num_cols),
            ("cat", cat_transformer, cat_cols),
        ]
    )

    # Full training pipeline
    clf = Pipeline(
        steps=[("preprocessor", preprocessor), ("classifier", RandomForestClassifier())]
    )

    # training
    clf.fit(X_train, y_train)

    # validation
    y_pred = clf.predict(X_val)
    print(f"Accuracy score {accuracy_score(list(y_val), list(y_pred))}")

    df_test.drop(columns=["Cabin"])
    preds = clf.predict(df_test)
    sample_submission = pd.read_csv(data_dir / "gender_submission.csv")
    sample_submission["Survived"] = preds
    sample_submission.to_csv("titanic_submission.csv", index=False)
//  [python]
def get_model_and_df(model: str = ModelTypes.LOGISTIC_REGRESSION, max_iter: int = 100) -> Tuple[Any, pd.DataFrame]:
    from sklearn import model_selection
    from sklearn.compose import ColumnTransformer
    from sklearn.feature_extraction.text import TfidfVectorizer
    from sklearn.preprocessing import Imputer
    from sklearn.linear_model import LogisticRegression
    from sklearn.pipeline import Pipeline
    from sklearn.preprocessing import OneHotEncoder, StandardScaler

    df = get_df()
    cat_cols = ["Pclass", "Sex", "SibSp", "Parch", "Embarked"]
    num_cols = ["PassengerId", "Age", "Fare"]
    text_cols = ["Name"]
    target = "Survived"

    # tfidf the text column
    text_transformer = Pipeline([("tfidf", TfidfVectorizer(lowercase=False, strip_accents=None))])

    # transform and scale the numeric columns
    num_transformer = Pipeline([("imputer", Imputer(strategy="median")), ("scaler", StandardScaler())])

    # one hot encode the categorical values
    cat_transormer = Pipeline(
        [
            ("imputer", Imputer(strategy="constant", fill_value="missing")),
            ("onehot", OneHotEncoder(handle_unknown="ignore", sparse_output=False)),
        ]
    )

    # Perform preprocessing of the columns with the above pipelines
    preprocessor = ColumnTransformer(
        transformers=[
            ("text", text_transformer, text_cols[0]),
            ("num", num_transformer, num_cols),
            ("cat", cat_transormer, cat_cols),
        ]
    )

    # Define the final ML Pipeline.
    if model.lower() == ModelTypes.LOGISTIC_REGRESSION.lower():
        clf = Pipeline(steps=[("preprocessor", preprocessor), ("classifier", LogisticRegression(max_iter=max_iter))])
    elif model.lower() == ModelTypes.LGBM_CLASSIFIER.lower():
        from lightgbm import LGBMClassifier

        clf = Pipeline(steps=[("preprocessor", preprocessor), ("classifier", LGBMClassifier(n_estimators=max_iter))])
    else:
        raise NotImplementedError(f"The model type '{model}' is not supported!")

    Y = df[target]
    X = df.drop(target, axis=1)
    X_train, X_test, Y_train, Y_test = model_selection.train_test_split(
        X, Y, test_size=0.50, random_state=30, stratify=Y
    )

    clf.fit(X_train, Y_train)

    test_data = pd.concat([X_test, Y_test], axis=1)

    return clf, test_data
//  [python]
def get_model_and_df(model: str = ModelTypes.LOGISTIC_REGRESSION, max_iter: int = 100) -> Tuple[Any, pd.DataFrame]:
    from sklearn import model_selection
    from sklearn.compose import ColumnTransformer
    from sklearn.feature_extraction.text import TfidfVectorizer
    from sklearn.impute import SimpleImputer
    from sklearn.linear_model import LogisticRegression
    from sklearn.pipeline import Pipeline
    from sklearn.preprocessing import OneHotEncoder, StandardScaler

    df = get_df()
    cat_cols = ["Pclass", "Sex", "SibSp", "Parch", "Embarked"]
    num_cols = ["PassengerId", "Age", "Fare"]
    text_cols = ["Name"]
    target = "Survived"

    # tfidf the text column
    text_transformer = Pipeline([("tfidf", TfidfVectorizer(lowercase=False, strip_accents=None))])

    # transform and scale the numeric columns
    num_transformer = Pipeline([("imputer", SimpleImputer(strategy="median")), ("scaler", StandardScaler())])

    # one hot encode the categorical values
    cat_transormer = Pipeline(
        [
            ("imputer", SimpleImputer(strategy="constant", fill_value="missing")),
            ("onehot", OneHotEncoder(handle_unknown="ignore", sparse_output=False)),
        ]
    )

    # Perform preprocessing of the columns with the above pipelines
    preprocessor = ColumnTransformer(
        transformers=[
            ("text", text_transformer, text_cols[0]),
            ("num", num_transformer, num_cols),
            ("cat", cat_transormer, cat_cols),
        ]
    )

    # Define the final ML Pipeline.
    if model.lower() == ModelTypes.LOGISTIC_REGRESSION.lower():
        clf = Pipeline(steps=[("preprocessor", preprocessor), ("classifier", LogisticRegression(max_iter=max_iter))])
    elif model.lower() == ModelTypes.LGBM_CLASSIFIER.lower():
        from lightgbm import LGBMClassifier

        clf = Pipeline(steps=[("preprocessor", preprocessor), ("classifier", LGBMClassifier(n_estimators=max_iter))])
    else:
        raise NotImplementedError(f"The model type '{model}' is not supported!")

    Y = df[target]
    X = df.drop(target, axis=1)
    X_train, X_test, Y_train, Y_test = model_selection.train_test_split(
        X, Y, test_size=0.50, random_state=30, stratify=Y
    )

    clf.fit(X_train, Y_train)

    test_data = pd.concat([X_test, Y_test], axis=1)

    return clf, test_data
//  [python]
def impute_nan(X, missing_value, random_state):
    """
    Missing value imputation
    """

    class imputer_:
        def transform(self, x):
            return x

        def fit(self, x):
            pass

    X = X[X.columns[~X.isnull().all()]]  # Remove columns w only nans
    if missing_value == "Zero":
        imp = Imputer(missing_values=np.nan, strategy="constant", fill_value=0)
    elif missing_value == "Mean":
        imp = Imputer(missing_values=np.nan, strategy="mean")
    elif missing_value == "Median":
        imp = Imputer(missing_values=np.nan, strategy="median")
    elif missing_value == "None":
        imp = imputer_()
    elif missing_value == "KNNImputer":
        imp = KNNImputer()
    else:
        raise NotImplementedError("Method {} not implemented".format(missing_value))

    imp.fit(X)
    X = pd.DataFrame(imp.transform(X), columns=X.columns)

    return X, imp
//  [python]
def impute_nan(X, missing_value, random_state):
    """
    Missing value imputation
    """

    class imputer_:
        def transform(self, x):
            return x

        def fit(self, x):
            pass

    X = X[X.columns[~X.isnull().all()]]  # Remove columns w only nans
    if missing_value == "Zero":
        imp = SimpleImputer(missing_values=np.nan, strategy="constant", fill_value=0)
    elif missing_value == "Mean":
        imp = SimpleImputer(missing_values=np.nan, strategy="mean")
    elif missing_value == "Median":
        imp = SimpleImputer(missing_values=np.nan, strategy="median")
    elif missing_value == "None":
        imp = imputer_()
    elif missing_value == "KNNImputer":
        imp = KNNImputer()
    else:
        raise NotImplementedError("Method {} not implemented".format(missing_value))

    imp.fit(X)
    X = pd.DataFrame(imp.transform(X), columns=X.columns)

    return X, imp
//  [python]
def _load_support_dataset():
  """Helper function to load and preprocess the SUPPORT dataset.

  The SUPPORT Dataset comes from the Vanderbilt University study
  to estimate survival for seriously ill hospitalized adults [1].

  Please refer to http://biostat.mc.vanderbilt.edu/wiki/Main/SupportDesc.
  for the original datasource.

  References
  ----------
  [1]: Knaus WA, Harrell FE, Lynn J et al. (1995): The SUPPORT prognostic
  model: Objective estimates of survival for seriously ill hospitalized
  adults. Annals of Internal Medicine 122:191-203.

  """

  data = pkgutil.get_data(__name__, 'datasets/support2.csv')
  data = pd.read_csv(io.BytesIO(data))
  x1 = data[['age', 'num.co', 'meanbp', 'wblc', 'hrt', 'resp', 'temp',
             'pafi', 'alb', 'bili', 'crea', 'sod', 'ph', 'glucose', 'bun',
             'urine', 'adlp', 'adls']]

  catfeats = ['sex', 'dzgroup', 'dzclass', 'income', 'race', 'ca']
  x2 = pd.get_dummies(data[catfeats])

  x = np.concatenate([x1, x2], axis=1)
  t = data['d.time'].values
  e = data['death'].values

  x = Imputer(missing_values=np.nan, strategy='mean').fit_transform(x)
  x = StandardScaler().fit_transform(x)

  remove = ~np.isnan(t)
  return x[remove], t[remove], e[remove]