jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def _fit_predict(cls, train_data, train_target, test_data, test_target):
        """Fit a model to the training data and make predictions for the test data."""
        del test_target  # delete argument which subclasses use but this method does not.
        unique_labels = np.unique(train_target)
        if len(unique_labels) == 1:
            predictions = np.full(len(test_data), unique_labels[0])
        else:
            ht = HyperTransformer()
            train_data = ht.fit_transform(train_data)
            test_data = ht.transform(test_data)

            test_data[np.isin(test_data, [np.inf, -np.inf])] = None
            train_data[np.isin(train_data, [np.inf, -np.inf])] = None

            model_kwargs = cls.MODEL_KWARGS.copy() if cls.MODEL_KWARGS else {}
            model = cls.MODEL(**model_kwargs)

            pipeline = Pipeline([
                ('imputer', Imputer()),
                ('scaler', RobustScaler()),
                ('model', model)
            ])

            pipeline.fit(train_data, train_target)

            predictions = pipeline.predict(test_data)

        return predictions
//  [python]
def _fit_predict(cls, train_data, train_target, test_data, test_target):
        """Fit a model to the training data and make predictions for the test data."""
        del test_target  # delete argument which subclasses use but this method does not.
        unique_labels = np.unique(train_target)
        if len(unique_labels) == 1:
            predictions = np.full(len(test_data), unique_labels[0])
        else:
            ht = HyperTransformer()
            train_data = ht.fit_transform(train_data)
            test_data = ht.transform(test_data)

            test_data[np.isin(test_data, [np.inf, -np.inf])] = None
            train_data[np.isin(train_data, [np.inf, -np.inf])] = None

            model_kwargs = cls.MODEL_KWARGS.copy() if cls.MODEL_KWARGS else {}
            model = cls.MODEL(**model_kwargs)

            pipeline = Pipeline([
                ('imputer', SimpleImputer()),
                ('scaler', RobustScaler()),
                ('model', model)
            ])

            pipeline.fit(train_data, train_target)

            predictions = pipeline.predict(test_data)

        return predictions
//  [python]
def test_pipeline_compatibility() -> None:
    """Check that MAPIE works on pipeline based on pandas dataframes"""
    X = pd.DataFrame(
        {
            "x_cat": ["A", "A", "B", "A", "A", "B"],
            "x_num": [0, 1, 1, 4, np.nan, 5],
            "y": [5, 7, 3, 9, 10, 8],
        }
    )
    y = pd.Series([5, 7, 3, 9, 10, 8])
    numeric_preprocessor = Pipeline(
        [
            ("imputer", Imputer(strategy="mean")),
        ]
    )
    categorical_preprocessor = Pipeline(
        steps=[("encoding", OneHotEncoder(handle_unknown="ignore"))]
    )
    preprocessor = ColumnTransformer(
        [
            ("cat", categorical_preprocessor, ["x_cat"]),
            ("num", numeric_preprocessor, ["x_num"]),
        ]
    )
    pipe = make_pipeline(preprocessor, LinearRegression())
    mapie = MapieRegressor(pipe)
    mapie.fit(X, y)
    mapie.predict(X)
//  [python]
def test_pipeline_compatibility() -> None:
    """Check that MAPIE works on pipeline based on pandas dataframes"""
    X = pd.DataFrame(
        {
            "x_cat": ["A", "A", "B", "A", "A", "B"],
            "x_num": [0, 1, 1, 4, np.nan, 5],
            "y": [5, 7, 3, 9, 10, 8],
        }
    )
    y = pd.Series([5, 7, 3, 9, 10, 8])
    numeric_preprocessor = Pipeline(
        [
            ("imputer", SimpleImputer(strategy="mean")),
        ]
    )
    categorical_preprocessor = Pipeline(
        steps=[("encoding", OneHotEncoder(handle_unknown="ignore"))]
    )
    preprocessor = ColumnTransformer(
        [
            ("cat", categorical_preprocessor, ["x_cat"]),
            ("num", numeric_preprocessor, ["x_num"]),
        ]
    )
    pipe = make_pipeline(preprocessor, LinearRegression())
    mapie = MapieRegressor(pipe)
    mapie.fit(X, y)
    mapie.predict(X)
//  [python]
def __init__(self, input_data) -> None:
        self._input_data = input_data
        self._logger.debug("Defining transformers.")
        numeric_features = list(feature_columns_names)
        numeric_features.remove("sex")
        numeric_transformer = Pipeline(
            steps=[("imputer", Imputer(strategy="median")), ("scaler", StandardScaler())]
        )

        categorical_features = ["sex"]
        categorical_transformer = Pipeline(
            steps=[
                ("imputer", Imputer(strategy="constant", fill_value="missing")),
                ("onehot", OneHotEncoder(handle_unknown="ignore")),
            ]
        )

        self._preprocess = ColumnTransformer(
            transformers=[
                ("num", numeric_transformer, numeric_features),
                ("cat", categorical_transformer, categorical_features),
            ]
        )

        self._logger.debug("Fitting transforms.")
        self._input_data_y = self._input_data.pop("rings")
        self._preprocess.fit(self._input_data)
//  [python]
def __init__(self, input_data) -> None:
        self._input_data = input_data
        self._logger.debug("Defining transformers.")
        numeric_features = list(feature_columns_names)
        numeric_features.remove("sex")
        numeric_transformer = Pipeline(
            steps=[("imputer", SimpleImputer(strategy="median")), ("scaler", StandardScaler())]
        )

        categorical_features = ["sex"]
        categorical_transformer = Pipeline(
            steps=[
                ("imputer", SimpleImputer(strategy="constant", fill_value="missing")),
                ("onehot", OneHotEncoder(handle_unknown="ignore")),
            ]
        )

        self._preprocess = ColumnTransformer(
            transformers=[
                ("num", numeric_transformer, numeric_features),
                ("cat", categorical_transformer, categorical_features),
            ]
        )

        self._logger.debug("Fitting transforms.")
        self._input_data_y = self._input_data.pop("rings")
        self._preprocess.fit(self._input_data)
//  [python]
def test_raw_timestamp_explanation(self, mimic_explainer):
        df = retrieve_dataset('insurance_claims.csv', na_values='?',
                              parse_dates=['policy_bind_date', 'incident_date'])
        label = 'fraud_reported'
        df_y = df[label]
        df_X = df.drop(columns=label)
        x_train, x_test, y_train, y_test = train_test_split(df_X, df_y, test_size=0.2, random_state=7)
        str_cols = df_X.select_dtypes(exclude=[np.number, np.datetime64]).columns.tolist()
        dt_cols = df_X.select_dtypes(include=[np.datetime64]).columns.tolist()
        numeric_cols = df_X.select_dtypes(include=[np.number]).columns.tolist()
        transforms_list = []
        for str_col in str_cols:
            transforms_list.append((str_col, Pipeline(steps=[
                ('imputer', Imputer(strategy='most_frequent')),
                ('ohe', OneHotEncoder(sparse=False))
                ]), [str_col]
            ))
        for numeric_col in numeric_cols:
            transforms_list.append((numeric_col, Pipeline(steps=[
                ('imputer', Imputer(strategy='mean')),
                ('scaler', StandardScaler())
                ]), [numeric_col]
            ))
        for dt_col in dt_cols:
            transforms_list.append((dt_col, Pipeline(steps=[
                ('scaler', StandardScaler())
                ]), [dt_col]
            ))
        transformations = ColumnTransformer(transforms_list)
        x_train_transformed = transformations.fit_transform(x_train)
        model = create_lightgbm_classifier(x_train_transformed, y_train)
        model_task = ModelTask.Classification
        features = df_X.columns.tolist()
        explainer = mimic_explainer(model, x_train, LGBMExplainableModel, transformations=transformations,
                                    features=features, model_task=model_task)
        explanation = explainer.explain_global(x_train)
        dashboard_pipeline = Pipeline(steps=[('preprocess', transformations), ('model', model)])
        ExplanationDashboard(explanation, dashboard_pipeline, dataset=x_train, true_y=y_train)
//  [python]
def test_raw_timestamp_explanation(self, mimic_explainer):
        df = retrieve_dataset('insurance_claims.csv', na_values='?',
                              parse_dates=['policy_bind_date', 'incident_date'])
        label = 'fraud_reported'
        df_y = df[label]
        df_X = df.drop(columns=label)
        x_train, x_test, y_train, y_test = train_test_split(df_X, df_y, test_size=0.2, random_state=7)
        str_cols = df_X.select_dtypes(exclude=[np.number, np.datetime64]).columns.tolist()
        dt_cols = df_X.select_dtypes(include=[np.datetime64]).columns.tolist()
        numeric_cols = df_X.select_dtypes(include=[np.number]).columns.tolist()
        transforms_list = []
        for str_col in str_cols:
            transforms_list.append((str_col, Pipeline(steps=[
                ('imputer', SimpleImputer(strategy='most_frequent')),
                ('ohe', OneHotEncoder(sparse=False))
                ]), [str_col]
            ))
        for numeric_col in numeric_cols:
            transforms_list.append((numeric_col, Pipeline(steps=[
                ('imputer', SimpleImputer(strategy='mean')),
                ('scaler', StandardScaler())
                ]), [numeric_col]
            ))
        for dt_col in dt_cols:
            transforms_list.append((dt_col, Pipeline(steps=[
                ('scaler', StandardScaler())
                ]), [dt_col]
            ))
        transformations = ColumnTransformer(transforms_list)
        x_train_transformed = transformations.fit_transform(x_train)
        model = create_lightgbm_classifier(x_train_transformed, y_train)
        model_task = ModelTask.Classification
        features = df_X.columns.tolist()
        explainer = mimic_explainer(model, x_train, LGBMExplainableModel, transformations=transformations,
                                    features=features, model_task=model_task)
        explanation = explainer.explain_global(x_train)
        dashboard_pipeline = Pipeline(steps=[('preprocess', transformations), ('model', model)])
        ExplanationDashboard(explanation, dashboard_pipeline, dataset=x_train, true_y=y_train)
//  [python]
def test_invalid_datatype_strategy(self):
        sk_trainable = sklearn.impute.SimpleImputer()
        with self.assertRaises(ValueError):
            sk_trainable.fit(self.tgt2adult["pandas"][0][0])
        rasl_trainable = RaslSimpleImputer()
        for tgt, dataset in self.tgt2adult.items():
            (train_X, _), (_, _) = dataset
            if tgt == "spark":
                # Skip test because of timeout!
                continue
            with self.assertRaises(ValueError):
                _ = rasl_trainable.fit(train_X)
//  [python]
def test_invalid_datatype_strategy(self):
        sk_trainable = SkSimpleImputer()
        with self.assertRaises(ValueError):
            sk_trainable.fit(self.tgt2adult["pandas"][0][0])
        rasl_trainable = RaslSimpleImputer()
        for tgt, dataset in self.tgt2adult.items():
            (train_X, _), (_, _) = dataset
            if tgt == "spark":
                # Skip test because of timeout!
                continue
            with self.assertRaises(ValueError):
                _ = rasl_trainable.fit(train_X)
//  [python]
def test_explain_raw_feats_titanic(self, tabular_explainer):
        titanic_url = ('https://raw.githubusercontent.com/amueller/'
                       'scipy-2017-sklearn/091d371/notebooks/datasets/titanic3.csv')
        data = pd.read_csv(titanic_url)
        # fill missing values
        data = data.fillna(method="ffill")
        data = data.fillna(method="bfill")
        numeric_features = ['age', 'fare']
        categorical_features = ['embarked', 'sex', 'pclass']
        y = data['survived'].values
        X = data[categorical_features + numeric_features]
        # Split data into train and test
        x_train, x_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

        def conv(X):
            if isinstance(X, pd.Series):
                return X.values
            return X

        many_to_one_transformer = FunctionTransformer(lambda x: conv(x.sum(axis=1)).reshape(-1, 1))
        many_to_many_transformer = FunctionTransformer(lambda x: np.hstack(
            (conv(np.prod(x, axis=1)).reshape(-1, 1), conv(np.prod(x, axis=1)**2).reshape(-1, 1))
        ))
        transformations = ColumnTransformer([
            ("age_fare_1", Pipeline(steps=[
                ('imputer', Imputer(strategy='median')),
                ('scaler', StandardScaler())
            ]), ["age", "fare"]),
            ("age_fare_2", many_to_one_transformer, ["age", "fare"]),
            ("age_fare_3", many_to_many_transformer, ["age", "fare"]),
            ("embarked", Pipeline(steps=[
                ("imputer", Imputer(strategy='constant', fill_value='missing')),
                ("encoder", OneHotEncoder(sparse=False))]), ["embarked"]),
            ("sex_pclass", OneHotEncoder(sparse=False), ["sex", "pclass"])
        ])
        clf = Pipeline(steps=[('preprocessor', transformations),
                              ('classifier', LogisticRegression(solver='lbfgs'))])
        clf.fit(x_train, y_train)
        explainer = tabular_explainer(clf.steps[-1][1],
                                      initialization_examples=x_train,
                                      features=x_train.columns,
                                      transformations=transformations,
                                      allow_all_transformations=True)
        explainer.explain_global(x_test)
        explainer.explain_local(x_test)
//  [python]
def test_explain_raw_feats_titanic(self, tabular_explainer):
        titanic_url = ('https://raw.githubusercontent.com/amueller/'
                       'scipy-2017-sklearn/091d371/notebooks/datasets/titanic3.csv')
        data = pd.read_csv(titanic_url)
        # fill missing values
        data = data.fillna(method="ffill")
        data = data.fillna(method="bfill")
        numeric_features = ['age', 'fare']
        categorical_features = ['embarked', 'sex', 'pclass']
        y = data['survived'].values
        X = data[categorical_features + numeric_features]
        # Split data into train and test
        x_train, x_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

        def conv(X):
            if isinstance(X, pd.Series):
                return X.values
            return X

        many_to_one_transformer = FunctionTransformer(lambda x: conv(x.sum(axis=1)).reshape(-1, 1))
        many_to_many_transformer = FunctionTransformer(lambda x: np.hstack(
            (conv(np.prod(x, axis=1)).reshape(-1, 1), conv(np.prod(x, axis=1)**2).reshape(-1, 1))
        ))
        transformations = ColumnTransformer([
            ("age_fare_1", Pipeline(steps=[
                ('imputer', SimpleImputer(strategy='median')),
                ('scaler', StandardScaler())
            ]), ["age", "fare"]),
            ("age_fare_2", many_to_one_transformer, ["age", "fare"]),
            ("age_fare_3", many_to_many_transformer, ["age", "fare"]),
            ("embarked", Pipeline(steps=[
                ("imputer", SimpleImputer(strategy='constant', fill_value='missing')),
                ("encoder", OneHotEncoder(sparse=False))]), ["embarked"]),
            ("sex_pclass", OneHotEncoder(sparse=False), ["sex", "pclass"])
        ])
        clf = Pipeline(steps=[('preprocessor', transformations),
                              ('classifier', LogisticRegression(solver='lbfgs'))])
        clf.fit(x_train, y_train)
        explainer = tabular_explainer(clf.steps[-1][1],
                                      initialization_examples=x_train,
                                      features=x_train.columns,
                                      transformations=transformations,
                                      allow_all_transformations=True)
        explainer.explain_global(x_test)
        explainer.explain_local(x_test)
//  [python]
def test_predict(self):
        self._fill_missing_value("age", 36.0, np.nan)
        (train_X_pd, train_y_pd), (test_X_pd, _test_y_pd) = self.tgt2adult["pandas"]
        num_columns = ["age", "fnlwgt", "education-num"]
        prefix = Map(columns={c: it[c] for c in num_columns})
        to_pd = Convert(astype="pandas")
        lr = LogisticRegression()
        imputer_args = {"strategy": "mean"}
        sk_trainable = prefix >> SkSimpleImputer(**imputer_args) >> lr
        sk_trained = sk_trainable.fit(train_X_pd, train_y_pd)
        sk_predicted = sk_trained.predict(test_X_pd)
        rasl_trainable = prefix >> RaslSimpleImputer(**imputer_args) >> to_pd >> lr
        for tgt, dataset in self.tgt2adult.items():
            (train_X, train_y), (test_X, _test_y) = dataset
            rasl_trained = rasl_trainable.fit(train_X, train_y)
            rasl_predicted = rasl_trained.predict(test_X)
            self.assertEqual(sk_predicted.shape, rasl_predicted.shape, tgt)
            self.assertEqual(sk_predicted.tolist(), rasl_predicted.tolist(), tgt)
//  [python]
def test_fit_transform_numeric_nan_missing(self):
        self._fill_missing_value("age", 36.0, np.nan)
        num_columns = ["age", "fnlwgt", "education-num"]
        prefix = Map(columns={c: it[c] for c in num_columns})

        hyperparams = [
            {"strategy": "mean"},
            {"strategy": "median"},
            {"strategy": "most_frequent"},
            {"strategy": "constant", "fill_value": 99},
        ]
        for hyperparam in hyperparams:
            rasl_imputer = RaslSimpleImputer(**hyperparam)
            sk_imputer = SkSimpleImputer(**hyperparam)
            rasl_trainable = prefix >> rasl_imputer
            sk_trainable = prefix >> sk_imputer
            sk_trained = sk_trainable.fit(self.tgt2adult["pandas"][0][0])
            sk_transformed = sk_trained.transform(self.tgt2adult["pandas"][1][0])
            sk_statistics_ = sk_trained.steps[-1][1].impl.statistics_
            for tgt, dataset in self.tgt2adult.items():
                (train_X, _), (test_X, _) = dataset
                rasl_trained = rasl_trainable.fit(train_X)
                # test the fit succeeded.
                rasl_statistics_ = rasl_trained.steps[-1][1].impl.statistics_

                self.assertEqual(
                    len(sk_statistics_), len(rasl_statistics_), (hyperparam, tgt)
                )
                for i in range(sk_statistics_.shape[0]):
                    self.assertAlmostEqual(
                        sk_statistics_[i],
                        rasl_statistics_[i],
                        msg=(i, hyperparam, tgt),
                        delta=10,
                    )

                rasl_transformed = rasl_trained.transform(test_X)
                if tgt == "spark":
                    self.assertEqual(get_index_name(rasl_transformed), "index")
                rasl_transformed = _ensure_pandas(rasl_transformed)
                self.assertEqual(sk_transformed.shape, rasl_transformed.shape, tgt)
                for row_idx in range(sk_transformed.shape[0]):
                    for col_idx in range(sk_transformed.shape[1]):
                        self.assertAlmostEqual(
                            sk_transformed[row_idx, col_idx],
                            rasl_transformed.iloc[row_idx, col_idx],
                            msg=(row_idx, col_idx, tgt),
                        )
                _check_trained_simple_imputer(self, sk_imputer, rasl_imputer, tgt)
//  [python]
def test_fit_transform_numeric_nonan_missing(self):
        self._fill_missing_value("age", 36.0, -1)
        num_columns = ["age", "fnlwgt", "education-num"]
        prefix = Map(columns={c: it[c] for c in num_columns})

        hyperparams = [
            {"strategy": "mean"},
            {"strategy": "median"},
            {"strategy": "most_frequent"},
            {"strategy": "constant", "fill_value": 99},
        ]
        for hyperparam in hyperparams:
            rasl_trainable = prefix >> RaslSimpleImputer(
                missing_values=-1, **hyperparam
            )
            sk_trainable = prefix >> SkSimpleImputer(missing_values=-1, **hyperparam)
            sk_trained = sk_trainable.fit(self.tgt2adult["pandas"][0][0])
            sk_transformed = sk_trained.transform(self.tgt2adult["pandas"][1][0])
            sk_statistics_ = sk_trained.get_last().impl.statistics_
            for tgt, dataset in self.tgt2adult.items():
                (train_X, _), (test_X, _) = dataset
                rasl_trained = rasl_trainable.fit(train_X)
                # test the fit succeeded.
                rasl_statistics_ = rasl_trained.get_last().impl.statistics_  # type: ignore
                self.assertEqual(len(sk_statistics_), len(rasl_statistics_), tgt)
                for sk_val, rasl_val in zip(sk_statistics_, rasl_statistics_):
                    self.assertAlmostEqual(
                        sk_val,
                        rasl_val,
                        msg=(list(sk_statistics_), list(rasl_statistics_), tgt),
                        delta=10,
                    )
                rasl_transformed = rasl_trained.transform(test_X)
                if tgt == "spark":
                    self.assertEqual(get_index_name(rasl_transformed), "index")
                rasl_transformed = _ensure_pandas(rasl_transformed)
                self.assertEqual(sk_transformed.shape, rasl_transformed.shape, tgt)
                for row_idx in range(sk_transformed.shape[0]):
                    for col_idx in range(sk_transformed.shape[1]):
                        self.assertEqual(
                            sk_transformed[row_idx, col_idx],
                            rasl_transformed.iloc[row_idx, col_idx],
                            (row_idx, col_idx, tgt),
                        )
//  [python]
def test_default_numeric_fill_value(self):
        self._fill_missing_value("age", 36.0, np.nan)
        num_columns = ["age", "fnlwgt", "education-num"]
        prefix = Map(columns={c: it[c] for c in num_columns})

        hyperparams = [{"strategy": "constant"}]
        for hyperparam in hyperparams:
            rasl_trainable = prefix >> RaslSimpleImputer(**hyperparam)
            sk_trainable = prefix >> SkSimpleImputer(**hyperparam)
            sk_trained = sk_trainable.fit(self.tgt2adult["pandas"][0][0])
            sk_transformed = sk_trained.transform(self.tgt2adult["pandas"][1][0])
            sk_statistics_ = sk_trained.steps[-1][1].impl.statistics_
            for tgt, dataset in self.tgt2adult.items():
                (train_X, _), (test_X, _) = dataset
                rasl_trained = rasl_trainable.fit(train_X)
                # test the fit succeeded.
                rasl_statistics_ = rasl_trained.steps[-1][1].impl.statistics_
                self.assertEqual(len(sk_statistics_), len(rasl_statistics_), tgt)
                self.assertEqual(list(sk_statistics_), list(rasl_statistics_), tgt)

                rasl_transformed = rasl_trained.transform(test_X)
                rasl_transformed = _ensure_pandas(rasl_transformed)
                self.assertEqual(sk_transformed.shape, rasl_transformed.shape, tgt)
                for row_idx in range(sk_transformed.shape[0]):
                    for col_idx in range(sk_transformed.shape[1]):
                        self.assertEqual(
                            sk_transformed[row_idx, col_idx],
                            rasl_transformed.iloc[row_idx, col_idx],
                            (row_idx, col_idx, tgt),
                        )
//  [python]
def test_multiple_modes_numeric(self):
        # Sklearn SimpleImputer says: for strategy `most_frequent`,
        # if there is more than one such value, only the smallest is returned.
        data = [[1, 10], [2, 14], [3, 15], [4, 15], [5, 14], [6, np.nan]]
        df = pd.DataFrame(data, columns=["Id", "Age"])
        hyperparam = {"strategy": "most_frequent"}
        sk_trainable = s.Imputer(**hyperparam)
        rasl_trainable = RaslSimpleImputer(**hyperparam)
        sk_trained = sk_trainable.fit(df)
        rasl_trained = rasl_trainable.fit(df)
        self.assertEqual(
            len(sk_trained.statistics_), len(rasl_trained.impl.statistics_), "pandas"
        )
        self.assertEqual([6, 15], list(rasl_trained.impl.statistics_), "pandas")
        spark_df = pandas2spark(df)

        rasl_trained = rasl_trainable.fit(spark_df)
        self.assertEqual(
            len(sk_trained.statistics_), len(rasl_trained.impl.statistics_), "spark"
        )
        self.assertIn(rasl_trained.impl.statistics_[1], [14, 15])
//  [python]
def test_multiple_modes_numeric(self):
        # Sklearn SimpleImputer says: for strategy `most_frequent`,
        # if there is more than one such value, only the smallest is returned.
        data = [[1, 10], [2, 14], [3, 15], [4, 15], [5, 14], [6, np.nan]]
        df = pd.DataFrame(data, columns=["Id", "Age"])
        hyperparam = {"strategy": "most_frequent"}
        sk_trainable = SkSimpleImputer(**hyperparam)
        rasl_trainable = RaslSimpleImputer(**hyperparam)
        sk_trained = sk_trainable.fit(df)
        rasl_trained = rasl_trainable.fit(df)
        self.assertEqual(
            len(sk_trained.statistics_), len(rasl_trained.impl.statistics_), "pandas"
        )
        self.assertEqual([6, 15], list(rasl_trained.impl.statistics_), "pandas")
        spark_df = pandas2spark(df)

        rasl_trained = rasl_trainable.fit(spark_df)
        self.assertEqual(
            len(sk_trained.statistics_), len(rasl_trained.impl.statistics_), "spark"
        )
        self.assertIn(rasl_trained.impl.statistics_[1], [14, 15])
//  [python]
def _make_sk_trainable(cls, final_est):
        from sklearn.compose import ColumnTransformer as SkColumnTransformer
        from sklearn.ensemble import RandomForestClassifier as SkRandomForestClassifier
        from sklearn.linear_model import SGDClassifier as SkSGDClassifier
        from sklearn.preprocessing import Imputer as SkSimpleImputer

        if final_est == "sgd":
            est = SkSGDClassifier(random_state=97)
        elif final_est == "rfc":
            est = SkRandomForestClassifier(random_state=97)
        else:
            assert False, final_est
        return sk_make_pipeline(
            SkColumnTransformer(
                [
                    (
                        "prep_cat",
                        SkOrdinalEncoder(
                            handle_unknown="use_encoded_value", unknown_value=-1
                        ),
                        cls.cat_columns,
                    ),
                    (
                        "prep_num",
                        sk_make_pipeline(
                            SkSimpleImputer(strategy="mean"),
                            SkMinMaxScaler(),
                            "passthrough",
                        ),
                        cls.num_columns,
                    ),
                ]
            ),
            est,
        )
//  [python]
def _make_sk_trainable(cls, final_est):
        from sklearn.compose import ColumnTransformer as SkColumnTransformer
        from sklearn.ensemble import RandomForestClassifier as SkRandomForestClassifier
        from sklearn.linear_model import SGDClassifier as SkSGDClassifier

        if final_est == "sgd":
            est = SkSGDClassifier(random_state=97)
        elif final_est == "rfc":
            est = SkRandomForestClassifier(random_state=97)
        else:
            assert False, final_est
        return sk_make_pipeline(
            SkColumnTransformer(
                [
                    (
                        "prep_cat",
                        SkOrdinalEncoder(
                            handle_unknown="use_encoded_value", unknown_value=-1
                        ),
                        cls.cat_columns,
                    ),
                    (
                        "prep_num",
                        sk_make_pipeline(
                            SkSimpleImputer(strategy="mean"),
                            SkMinMaxScaler(),
                            "passthrough",
                        ),
                        cls.num_columns,
                    ),
                ]
            ),
            est,
        )
//  [python]
def test_multiple_modes_string(self):
        # Sklearn SimpleImputer says: for strategy `most_frequent`,
        # if there is more than one such value, only the smallest is returned.
        data = [
            ["a", "t"],
            ["b", "f"],
            ["b", "m"],
            ["c", "f"],
            ["c", "m"],
            ["f", "missing"],
        ]
        df = pd.DataFrame(data, columns=["Id", "Gender"])
        hyperparam = {"strategy": "most_frequent", "missing_values": "missing"}
        sk_trainable = s.Imputer(**hyperparam)
        rasl_trainable = RaslSimpleImputer(**hyperparam)
        sk_trained = sk_trainable.fit(df)
        rasl_trained = rasl_trainable.fit(df)
        self.assertEqual(
            len(sk_trained.statistics_), len(rasl_trained.impl.statistics_), "pandas"
        )
        self.assertEqual(
            list(["c", "m"]), list(rasl_trained.impl.statistics_), "pandas"
        )

        spark_df = pandas2spark(df)

        rasl_trained = rasl_trainable.fit(spark_df)
        self.assertEqual(
            len(sk_trained.statistics_), len(rasl_trained.impl.statistics_), "spark"
        )
        self.assertIn(rasl_trained.impl.statistics_[1], ["f", "m"])
//  [python]
def test_multiple_modes_string(self):
        # Sklearn SimpleImputer says: for strategy `most_frequent`,
        # if there is more than one such value, only the smallest is returned.
        data = [
            ["a", "t"],
            ["b", "f"],
            ["b", "m"],
            ["c", "f"],
            ["c", "m"],
            ["f", "missing"],
        ]
        df = pd.DataFrame(data, columns=["Id", "Gender"])
        hyperparam = {"strategy": "most_frequent", "missing_values": "missing"}
        sk_trainable = SkSimpleImputer(**hyperparam)
        rasl_trainable = RaslSimpleImputer(**hyperparam)
        sk_trained = sk_trainable.fit(df)
        rasl_trained = rasl_trainable.fit(df)
        self.assertEqual(
            len(sk_trained.statistics_), len(rasl_trained.impl.statistics_), "pandas"
        )
        self.assertEqual(
            list(["c", "m"]), list(rasl_trained.impl.statistics_), "pandas"
        )

        spark_df = pandas2spark(df)

        rasl_trained = rasl_trainable.fit(spark_df)
        self.assertEqual(
            len(sk_trained.statistics_), len(rasl_trained.impl.statistics_), "spark"
        )
        self.assertIn(rasl_trained.impl.statistics_[1], ["f", "m"])
//  [python]
def test_default_string_fill_value(self):
        self._fill_missing_value("education", "Prof-school", np.nan)

        str_columns = ["workclass", "education", "capital-gain"]
        prefix = Map(columns={c: it[c] for c in str_columns})

        hyperparams = [{"strategy": "constant"}]
        for hyperparam in hyperparams:
            rasl_trainable = prefix >> RaslSimpleImputer(**hyperparam)
            sk_trainable = prefix >> SkSimpleImputer(**hyperparam)
            sk_trained = sk_trainable.fit(self.tgt2adult["pandas"][0][0])
            sk_statistics_ = sk_trained.steps[-1][1].impl.statistics_
            for tgt, dataset in self.tgt2adult.items():
                (train_X, _), (test_X, _) = dataset
                rasl_trained = rasl_trainable.fit(train_X)
                # test the fit succeeded.
                rasl_statistics_ = rasl_trained.steps[-1][1].impl.statistics_
                self.assertEqual(len(sk_statistics_), len(rasl_statistics_), tgt)
                self.assertEqual(list(sk_statistics_), list(rasl_statistics_), tgt)

                rasl_transformed = rasl_trained.transform(test_X)
                rasl_transformed = _ensure_pandas(rasl_transformed)
                # Note that for this test case, the output of sklearn transform does not
                # match rasl transform. There is at least one row which has a None
                # value and pandas replace treats it as nan and replaces it.
                # Sklearn which uses numpy does not replace a None.
                # So we just test that `missing_value` is the default assigned.
                self.assertEqual(rasl_transformed.iloc[1, 1], "missing_value")
//  [python]
def __load_dataset__(self):
        df = pd.io.stata.read_stata(self.train_file)
        orderings = []
        features = []
        for row in df.itertuples():
            orderings.append(row[4:8])
            context_feature = [float(i) if i != "." else np.NAN for i in row[13:33]]
            features.append(context_feature)
        X = np.array(features)
        X = Imputer().fit_transform(X)
        X = np.array([np.log(np.array(X[:, i]) + 1) for i in range(len(features[0]))])
        X = np.array(X.T)
        self.X = StandardScaler().fit_transform(X)
        orderings = np.array(orderings) - 1
        self.Y = ranking_ordering_conversion(orderings)
        self.__check_dataset_validity__()
//  [python]
def __load_dataset__(self):
        df = pd.io.stata.read_stata(self.train_file)
        orderings = []
        features = []
        for row in df.itertuples():
            orderings.append(row[4:8])
            context_feature = [float(i) if i != "." else np.NAN for i in row[13:33]]
            features.append(context_feature)
        X = np.array(features)
        X = SimpleImputer().fit_transform(X)
        X = np.array([np.log(np.array(X[:, i]) + 1) for i in range(len(features[0]))])
        X = np.array(X.T)
        self.X = StandardScaler().fit_transform(X)
        orderings = np.array(orderings) - 1
        self.Y = ranking_ordering_conversion(orderings)
        self.__check_dataset_validity__()
//  [python]
def reduce_data(X, n_components=20, perplexity=30, random_state=42, imputer_strategy="mean", scaler=None,
                reduction_algorithm="PCA", return_scaled=False, return_suffix=False, return_meta=False, **kw):
    """ Reduce input data to 2 components, using a combination of PCA/ICA and TSNE, first imputing missing values and
         scaling data with an input scaler. """
    from sklearn.preprocessing import Imputer
    from sklearn.preprocessing import MinMaxScaler
    l = kw.get('logger', null_logger)
    n, n_cols, p, ra = n_components, len(X.columns), perplexity, reduction_algorithm
    rs, suffix, metadata = 42, "", {}
    # impute missing values and scale data
    l.debug(f"imputing values using strategy {imputer_strategy}...")
    X = Imputer(missing_values=np.nan, strategy=imputer_strategy).fit_transform(X)
    l.debug(f"scaling data with {(scaler or MinMaxScaler).__name__}...")
    Xs = X = (scaler or MinMaxScaler)().fit_transform(X)
    # preprocess data with a PCA with n components to reduce the high dimensionality (better performance)
    if n < n_cols:
        from sklearn.decomposition import FastICA, PCA
        a = {'ICA': FastICA, 'PCA': PCA}[ra](n, random_state=rs() if is_function(rs) else rs)
        suffix += f"_{ra.lower()}{n}"
        metadata[ra + ':n'] = n
        l.debug(f"reducing data to {n} components with {ra}...")
        if 'target' in kw:
            a.fit(X, kw['target'])
            X = a.transform(X)
        else:
            X = a.fit_transform(X)
    # now reduce the n components to 2 dimensions with t-SNE (better results but less performance) if relevant
    if n == 2 and p != 30:
        l.warning("setting the perplexity when using PCA with 2 components has no effect because TSNE is not used")
    if n > 2:
        from sklearn.manifold import TSNE
        l.debug(f"reducing data to 2 components with TSNE...")
        X = TSNE(2, random_state=rs() if is_function(rs) else rs, perplexity=p).fit_transform(X)
        suffix += f"_tsne2-p{p}"
        metadata['TSNE:n'] = 2
        metadata['perplexity'] = p
    # return result(s) accordingly
    r = (X, )
    if return_scaled:
        r += (Xs, )
    if return_suffix:
        r += (suffix, )
    if return_meta:
        r += (metadata, )
    return r[0] if len(r) == 1 else r
//  [python]
def reduce_data(X, n_components=20, perplexity=30, random_state=42, imputer_strategy="mean", scaler=None,
                reduction_algorithm="PCA", return_scaled=False, return_suffix=False, return_meta=False, **kw):
    """ Reduce input data to 2 components, using a combination of PCA/ICA and TSNE, first imputing missing values and
         scaling data with an input scaler. """
    from sklearn.impute import SimpleImputer
    from sklearn.preprocessing import MinMaxScaler
    l = kw.get('logger', null_logger)
    n, n_cols, p, ra = n_components, len(X.columns), perplexity, reduction_algorithm
    rs, suffix, metadata = 42, "", {}
    # impute missing values and scale data
    l.debug(f"imputing values using strategy {imputer_strategy}...")
    X = SimpleImputer(missing_values=np.nan, strategy=imputer_strategy).fit_transform(X)
    l.debug(f"scaling data with {(scaler or MinMaxScaler).__name__}...")
    Xs = X = (scaler or MinMaxScaler)().fit_transform(X)
    # preprocess data with a PCA with n components to reduce the high dimensionality (better performance)
    if n < n_cols:
        from sklearn.decomposition import FastICA, PCA
        a = {'ICA': FastICA, 'PCA': PCA}[ra](n, random_state=rs() if is_function(rs) else rs)
        suffix += f"_{ra.lower()}{n}"
        metadata[ra + ':n'] = n
        l.debug(f"reducing data to {n} components with {ra}...")
        if 'target' in kw:
            a.fit(X, kw['target'])
            X = a.transform(X)
        else:
            X = a.fit_transform(X)
    # now reduce the n components to 2 dimensions with t-SNE (better results but less performance) if relevant
    if n == 2 and p != 30:
        l.warning("setting the perplexity when using PCA with 2 components has no effect because TSNE is not used")
    if n > 2:
        from sklearn.manifold import TSNE
        l.debug(f"reducing data to 2 components with TSNE...")
        X = TSNE(2, random_state=rs() if is_function(rs) else rs, perplexity=p).fit_transform(X)
        suffix += f"_tsne2-p{p}"
        metadata['TSNE:n'] = 2
        metadata['perplexity'] = p
    # return result(s) accordingly
    r = (X, )
    if return_scaled:
        r += (Xs, )
    if return_suffix:
        r += (suffix, )
    if return_meta:
        r += (metadata, )
    return r[0] if len(r) == 0 else r
//  [python]
def test_valid_partial_fit(self):
        self._fill_missing_value("age", 36.0, -1)
        num_columns = ["age", "fnlwgt", "education-num"]
        prefix = Map(columns={c: it[c] for c in num_columns})

        hyperparams = [
            {"strategy": "mean"},
            {"strategy": "constant", "fill_value": 99},
        ]
        for hyperparam in hyperparams:
            rasl_trainable = prefix >> RaslSimpleImputer(
                missing_values=-1, **hyperparam
            )
            sk_trainable = prefix >> SkSimpleImputer(missing_values=-1, **hyperparam)
            sk_trained = sk_trainable.fit(self.tgt2adult["pandas"][0][0])
            sk_transformed = sk_trained.transform(self.tgt2adult["pandas"][1][0])
            sk_statistics_ = sk_trained.get_last().impl.statistics_
            (train_X, _), (test_X, _) = self.tgt2adult["pandas"]
            data1_pandas = train_X.iloc[:10]
            data2_pandas = train_X.iloc[10:100]
            data3_pandas = train_X.iloc[100:]
            test_X_pandas = test_X

            for tgt in self.tgt2adult.keys():
                if tgt == "pandas":
                    data1 = data1_pandas
                    data2 = data2_pandas
                    data3 = data3_pandas
                    test_X = test_X_pandas
                elif tgt == "spark":
                    data1 = pandas2spark(data1_pandas)  # type:ignore
                    data2 = pandas2spark(data2_pandas)  # type:ignore
                    data3 = pandas2spark(data3_pandas)  # type:ignore
                    test_X = pandas2spark(test_X_pandas)  # type:ignore
                else:
                    assert False
                rasl_trainable = prefix >> RaslSimpleImputer(
                    missing_values=-1, **hyperparam
                )
                rasl_trained = rasl_trainable.partial_fit(data1)
                rasl_trained = rasl_trained.partial_fit(data2)
                rasl_trained = rasl_trained.partial_fit(data3)
                # test the fit succeeded.
                rasl_statistics_ = rasl_trained.get_last().impl.statistics_  # type: ignore

                self.assertEqual(len(sk_statistics_), len(rasl_statistics_), tgt)
                for sk_stats, rasl_stats in zip(sk_statistics_, rasl_statistics_):
                    self.assertEqual(sk_stats, rasl_stats)

                rasl_transformed = rasl_trained.transform(test_X)
                rasl_transformed = _ensure_pandas(rasl_transformed)
                self.assertEqual(sk_transformed.shape, rasl_transformed.shape, tgt)
                for row_idx in range(sk_transformed.shape[0]):
                    for col_idx in range(sk_transformed.shape[1]):
                        self.assertEqual(
                            sk_transformed[row_idx, col_idx],
                            rasl_transformed.iloc[row_idx, col_idx],
                            (row_idx, col_idx, tgt),
                        )
//  [python]
def load_data(save_directory) -> AbstractDataSource:
        h5_file = os.path.join(save_directory, "ccle_protein_quantification.h5")
        if not os.path.exists(h5_file):
            gz_file_path = os.path.join(save_directory, "ccle_protein_quantification.csv.gz")
            if not os.path.exists(gz_file_path):
                sp.download_streamed(CCLEProteinQuantification.FILE_URL, gz_file_path)
            df = pd.read_csv(gz_file_path, compression='gzip',
                             index_col="Gene_Symbol")
            excluded_columns = ["Protein_Id", "Description", "Group_ID", "Uniprot", "Uniprot_Acc"]
            excluded_indices = [df.columns.values.tolist().index(name) for name in excluded_columns]
            included_indices = list(sorted(set(list(range(len(df.columns)))) - set(excluded_indices)))
            data = df.values[:, included_indices].astype(float)

            si = Imputer(missing_values=float("nan"), strategy='mean')
            data = si.fit_transform(data)

            row_names = df.index.values.tolist()
            name_missing_indices = np.where(list(map(lambda x: isinstance(x, float) and np.isnan(x), row_names)))[0]
            for idx in name_missing_indices:
                row_names[idx] = ""

            name_converter = HGNCNames(save_directory)
            row_names = name_converter.update_outdated_gene_names(row_names)
            row_names, idx_start = np.unique(sorted(row_names), return_index=True)
            data = data[idx_start]
            col_names = df.columns.values[included_indices].tolist()

            HDF5Tools.save_h5_file(h5_file,
                                   data,
                                   "ccle_protein_quantification",
                                   column_names=col_names,
                                   row_names=row_names)
        data_source = HDF5DataSource(h5_file, fill_missing_value=0,
                                     duplicate_merge_strategy=sp.FirstEntryMergeStrategy())
        return data_source
//  [python]
def load_data(save_directory) -> AbstractDataSource:
        h5_file = os.path.join(save_directory, "ccle_protein_quantification.h5")
        if not os.path.exists(h5_file):
            gz_file_path = os.path.join(save_directory, "ccle_protein_quantification.csv.gz")
            if not os.path.exists(gz_file_path):
                sp.download_streamed(CCLEProteinQuantification.FILE_URL, gz_file_path)
            df = pd.read_csv(gz_file_path, compression='gzip',
                             index_col="Gene_Symbol")
            excluded_columns = ["Protein_Id", "Description", "Group_ID", "Uniprot", "Uniprot_Acc"]
            excluded_indices = [df.columns.values.tolist().index(name) for name in excluded_columns]
            included_indices = list(sorted(set(list(range(len(df.columns)))) - set(excluded_indices)))
            data = df.values[:, included_indices].astype(float)

            si = SimpleImputer(missing_values=float("nan"), strategy='mean')
            data = si.fit_transform(data)

            row_names = df.index.values.tolist()
            name_missing_indices = np.where(list(map(lambda x: isinstance(x, float) and np.isnan(x), row_names)))[0]
            for idx in name_missing_indices:
                row_names[idx] = ""

            name_converter = HGNCNames(save_directory)
            row_names = name_converter.update_outdated_gene_names(row_names)
            row_names, idx_start = np.unique(sorted(row_names), return_index=True)
            data = data[idx_start]
            col_names = df.columns.values[included_indices].tolist()

            HDF5Tools.save_h5_file(h5_file,
                                   data,
                                   "ccle_protein_quantification",
                                   column_names=col_names,
                                   row_names=row_names)
        data_source = HDF5DataSource(h5_file, fill_missing_value=0,
                                     duplicate_merge_strategy=sp.FirstEntryMergeStrategy())
        return data_source
//  [python]
def create_preprocessing_pipeline(X):
    """
    perform preprocessing on X:
        - drop diag_1_desc
        - convert all diag vars to string
        - convert race of obj
        - median impute and scale numeric features
        - constant impute and one-hot encode categorical features
    """
    # Drop diag_1_desc columns
    try:
        X.drop(["diag_1_desc"], axis=1, inplace=True)
    except:
        pass

    X["race"] = X["race"].astype("object")
    X["diag_1"] = X["diag_1"].astype("str")
    X["diag_2"] = X["diag_2"].astype("str")
    X["diag_3"] = X["diag_3"].astype("str")

    # Preprocessing for numerical features
    numeric_features = list(X.select_dtypes("int64").columns)
    numeric_transformer = Pipeline(
        steps=[("imputer", Imputer(strategy="median")), ("scaler", StandardScaler())]
    )

    # Preprocessing for categorical features
    categorical_features = list(X.select_dtypes("object").columns)
    categorical_transformer = Pipeline(
        steps=[
            ("imputer", Imputer(strategy="constant", fill_value="missing")),
            ("onehot", OneHotEncoder(handle_unknown="ignore")),
        ]
    )

    # Preprocessor with all of the steps
    preprocessor = ColumnTransformer(
        transformers=[
            ("num", numeric_transformer, numeric_features),
            ("cat", categorical_transformer, categorical_features),
        ]
    )

    # Full preprocessing pipeline
    pipeline = Pipeline(steps=[("preprocessor", preprocessor)])
    return pipeline
//  [python]
def create_preprocessing_pipeline(X):
    """
    perform preprocessing on X:
        - drop diag_1_desc
        - convert all diag vars to string
        - convert race of obj
        - median impute and scale numeric features
        - constant impute and one-hot encode categorical features
    """
    # Drop diag_1_desc columns
    try:
        X.drop(["diag_1_desc"], axis=1, inplace=True)
    except:
        pass

    X["race"] = X["race"].astype("object")
    X["diag_1"] = X["diag_1"].astype("str")
    X["diag_2"] = X["diag_2"].astype("str")
    X["diag_3"] = X["diag_3"].astype("str")

    # Preprocessing for numerical features
    numeric_features = list(X.select_dtypes("int64").columns)
    numeric_transformer = Pipeline(
        steps=[("imputer", SimpleImputer(strategy="median")), ("scaler", StandardScaler())]
    )

    # Preprocessing for categorical features
    categorical_features = list(X.select_dtypes("object").columns)
    categorical_transformer = Pipeline(
        steps=[
            ("imputer", SimpleImputer(strategy="constant", fill_value="missing")),
            ("onehot", OneHotEncoder(handle_unknown="ignore")),
        ]
    )

    # Preprocessor with all of the steps
    preprocessor = ColumnTransformer(
        transformers=[
            ("num", numeric_transformer, numeric_features),
            ("cat", categorical_transformer, categorical_features),
        ]
    )

    # Full preprocessing pipeline
    pipeline = Pipeline(steps=[("preprocessor", preprocessor)])
    return pipeline