jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def test_cross_val_score_allow_nans():
    # Check that cross_val_score allows input data with NaNs
    X = np.arange(200, dtype=np.float64).reshape(10, -1)
    X[2, :] = np.nan
    y = np.repeat([0, 1], X.shape[0] / 2)
    p = Pipeline([
        ('imputer', SimpleImputer(strategy='mean', missing_values=np.nan)),
        ('classifier', MockClassifier()),
    ])
    cval.cross_val_score(p, X, y, cv=5)
//  [python]
def test_permutation_test_score_allow_nans():
    # Check that permutation_test_score allows input data with NaNs
    X = np.arange(200, dtype=np.float64).reshape(10, -1)
    X[2, :] = np.nan
    y = np.repeat([0, 1], X.shape[0] / 2)
    p = Pipeline([
        ('imputer', Imputer(strategy='mean', missing_values='NaN')),
        ('classifier', MockClassifier()),
    ])
    cval.permutation_test_score(p, X, y, cv=5)
//  [python]
def test_permutation_test_score_allow_nans():
    # Check that permutation_test_score allows input data with NaNs
    X = np.arange(200, dtype=np.float64).reshape(10, -1)
    X[2, :] = np.nan
    y = np.repeat([0, 1], X.shape[0] / 2)
    p = Pipeline([
        ('imputer', SimpleImputer(strategy='mean', missing_values=np.nan)),
        ('classifier', MockClassifier()),
    ])
    cval.permutation_test_score(p, X, y, cv=5)
//  [python]
def _fit_gmm(self, data_points, device=None):
        """fit a GMM model with the data of the memory bank to find relative better 
        cost threshold to filter the pseudo labels.   
        Args:
            data_points: (tensor), the instance-level cost data points. 
        """
      
        pos_cost_gmm = data_points
        if len(pos_cost_gmm) == 0:
            return 0
            
        # initialization of the GMM model
        pos_cost_gmm, sort_inds = pos_cost_gmm.sort()   # from low to high according the cost
        pos_cost_gmm = pos_cost_gmm.view(-1, 1).cpu().numpy()
        min_cost, max_cost = pos_cost_gmm.min(), pos_cost_gmm.max()
        means_init = np.array([min_cost, max_cost]).reshape(2, 1)
        weights_init = np.array([0.5, 0.5])
        precisions_init = np.array([1.0, 1.0]).reshape(2, 1, 1)  # full
        if self.covariance_type == 'spherical':
            precisions_init = precisions_init.reshape(2)
        elif self.covariance_type == 'diag':
            precisions_init = precisions_init.reshape(2, 1)
        elif self.covariance_type == 'tied':
            precisions_init = np.array([[1.0]])
        if skm is None:
            raise ImportError('Please run "pip install sklearn" '
                            'to install sklearn first.')

        # do the GMM model fit
        gmm = skm.mixture.GaussianMixture(
            2,
            weights_init=weights_init,
            means_init=means_init,
            precisions_init=precisions_init,
            covariance_type=self.covariance_type,
            reg_covar=1e-5)
        
        if len(pos_cost_gmm) < 2:
            return pos_cost_gmm[0]
        
        gmm.fit(pos_cost_gmm)
       
        gmm_assignment = gmm.predict(pos_cost_gmm)
        scores = gmm.score_samples(pos_cost_gmm)
        gmm_assignment = torch.from_numpy(gmm_assignment).to(device)
        scores = torch.from_numpy(scores).to(device)
        pseudo_mask = gmm_assignment == 0
        cost_thr = None
        if pseudo_mask.nonzero().numel():
            _, pseudo_thr_ind = scores[pseudo_mask].topk(1)
            cost_thr = pos_cost_gmm[pseudo_mask][pseudo_thr_ind]
        if cost_thr is not None:
            return cost_thr
        else:
            # import ipdb;ipdb.set_trace()
            pseudo_mask = gmm_assignment == 1
            _, pseudo_thr_ind = scores[pseudo_mask].topk(1)
            cost_thr = pos_cost_gmm[pseudo_mask][pseudo_thr_ind]
            return cost_thr
//  [python]
def _fit_gmm(self, data_points, device=None):
        """fit a GMM model with the data of the memory bank to find relative better 
        cost threshold to filter the pseudo labels.   
        Args:
            data_points: (tensor), the instance-level cost data points. 
        """
      
        pos_cost_gmm = data_points
        if len(pos_cost_gmm) == 0:
            return 0
            
        # initialization of the GMM model
        pos_cost_gmm, sort_inds = pos_cost_gmm.sort()   # from low to high according the cost
        pos_cost_gmm = pos_cost_gmm.view(-1, 1).cpu().numpy()
        min_cost, max_cost = pos_cost_gmm.min(), pos_cost_gmm.max()
        means_init = np.array([min_cost, max_cost]).reshape(2, 1)
        weights_init = np.array([0.5, 0.5])
        precisions_init = np.array([1.0, 1.0]).reshape(2, 1, 1)  # full
        if self.covariance_type == 'spherical':
            precisions_init = precisions_init.reshape(2)
        elif self.covariance_type == 'diag':
            precisions_init = precisions_init.reshape(2, 1)
        elif self.covariance_type == 'tied':
            precisions_init = np.array([[1.0]])
        if skm is None:
            raise ImportError('Please run "pip install sklearn" '
                            'to install sklearn first.')

        # do the GMM model fit
        gmm = skm.GaussianMixture(
            2,
            weights_init=weights_init,
            means_init=means_init,
            precisions_init=precisions_init,
            covariance_type=self.covariance_type,
            reg_covar=1e-5)
        
        if len(pos_cost_gmm) < 2:
            return pos_cost_gmm[0]
        
        gmm.fit(pos_cost_gmm)
       
        gmm_assignment = gmm.predict(pos_cost_gmm)
        scores = gmm.score_samples(pos_cost_gmm)
        gmm_assignment = torch.from_numpy(gmm_assignment).to(device)
        scores = torch.from_numpy(scores).to(device)
        pseudo_mask = gmm_assignment == 0
        cost_thr = None
        if pseudo_mask.nonzero().numel():
            _, pseudo_thr_ind = scores[pseudo_mask].topk(1)
            cost_thr = pos_cost_gmm[pseudo_mask][pseudo_thr_ind]
        if cost_thr is not None:
            return cost_thr
        else:
            # import ipdb;ipdb.set_trace()
            pseudo_mask = gmm_assignment == 1
            _, pseudo_thr_ind = scores[pseudo_mask].topk(1)
            cost_thr = pos_cost_gmm[pseudo_mask][pseudo_thr_ind]
            return cost_thr
//  [python]
def f2():
    iris = load_iris()

    print(np.vstack((np.array([np.nan, np.nan, np.nan, np.nan]), iris.data)))
    print("=="*64)

    # 空值填充
    # missing_values : integer or "NaN", optional (default="NaN")
    # 参数strategy为缺失值填充方式，默认为mean（均值）
    print(Imputer().fit_transform(np.vstack((np.array([np.nan, np.nan, np.nan, np.nan]), iris.data))))
    print("=="*64)

    # 常见的数据变换有基于多项式的、基于指数函数的、基于对数函数的，也可以自定义函数
    print(FunctionTransformer(np.log1p).fit_transform(iris.data))
    print("=="*64)
//  [python]
def f2():
    iris = load_iris()

    print(np.vstack((np.array([np.nan, np.nan, np.nan, np.nan]), iris.data)))
    print("=="*64)

    # 空值填充
    # missing_values : integer or "NaN", optional (default="NaN")
    # 参数strategy为缺失值填充方式，默认为mean（均值）
    print(SimpleImputer().fit_transform(np.vstack((np.array([np.nan, np.nan, np.nan, np.nan]), iris.data))))
    print("=="*64)

    # 常见的数据变换有基于多项式的、基于指数函数的、基于对数函数的，也可以自定义函数
    print(FunctionTransformer(np.log1p).fit_transform(iris.data))
    print("=="*64)
//  [python]
def test_grid_search_allows_nans():
    # Test GridSearchCV with Imputer
    X = np.arange(20, dtype=np.float64).reshape(5, -1)
    X[2, :] = np.nan
    y = [0, 0, 1, 1, 1]
    p = Pipeline([
        ('imputer', Imputer(strategy='mean', missing_values='NaN')),
        ('classifier', MockClassifier()),
    ])
    GridSearchCV(p, {'classifier__foo_param': [1, 2, 3]}, cv=2).fit(X, y)
//  [python]
def test_grid_search_allows_nans():
    # Test GridSearchCV with Imputer
    X = np.arange(20, dtype=np.float64).reshape(5, -1)
    X[2, :] = np.nan
    y = [0, 0, 1, 1, 1]
    p = Pipeline([
        ('imputer', SimpleImputer(strategy='mean', missing_values=np.nan)),
        ('classifier', MockClassifier()),
    ])
    GridSearchCV(p, {'classifier__foo_param': [1, 2, 3]}, cv=2).fit(X, y)
//  [python]
def impute_and_scale(df, scaling="std"):
    """Impute missing values with mean and scale data included in pandas dataframe.

    Parameters
    ----------
    df : pandas dataframe
        dataframe to impute and scale
    scaling : 'maxabs' [-1,1], 'minmax' [0,1], 'std', or None, optional (default 'std')
        type of scaling to apply
    """

    df = df.dropna(axis=1, how="all")

    imputer = Imputer(strategy="mean")
    mat = imputer.fit_transform(df)

    if scaling is None or scaling.lower() == "none":
        return pd.DataFrame(mat, columns=df.columns)

    if scaling == "maxabs":
        scaler = MaxAbsScaler()
    elif scaling == "minmax":
        scaler = MinMaxScaler()
    else:
        scaler = StandardScaler()

    mat = scaler.fit_transform(mat)

    df = pd.DataFrame(mat, columns=df.columns)

    return df
//  [python]
def impute_and_scale(df, scaling="std"):
    """Impute missing values with mean and scale data included in pandas dataframe.

    Parameters
    ----------
    df : pandas dataframe
        dataframe to impute and scale
    scaling : 'maxabs' [-1,1], 'minmax' [0,1], 'std', or None, optional (default 'std')
        type of scaling to apply
    """

    df = df.dropna(axis=1, how="all")

    imputer = SimpleImputer(strategy="mean")
    mat = imputer.fit_transform(df)

    if scaling is None or scaling.lower() == "none":
        return pd.DataFrame(mat, columns=df.columns)

    if scaling == "maxabs":
        scaler = MaxAbsScaler()
    elif scaling == "minmax":
        scaler = MinMaxScaler()
    else:
        scaler = StandardScaler()

    mat = scaler.fit_transform(mat)

    df = pd.DataFrame(mat, columns=df.columns)

    return df
//  [python]
def test_imputer_float_inputs(self):
        model = Imputer(missing_values="NaN", strategy="mean", axis=0)
        data = [[1, 2], [np.nan, 3], [7, 6]]
        model.fit(data)

        model_onnx = convert_sklearn(
            model, "scikit-learn imputer", [("input", FloatTensorType([None, 2]))]
        )
        self.assertTrue(model_onnx.graph.node is not None)

        # should contain only node
        self.assertEqual(len(model_onnx.graph.node), 1)

        # last node should contain the Imputer
        outputs = model_onnx.graph.output
        self.assertEqual(len(outputs), 1)
        self.assertEqual(outputs[0].type.tensor_type.shape.dim[-1].dim_value, 2)
        dump_data_and_model(
            np.array(data, dtype=np.float32),
            model,
            model_onnx,
            basename="SklearnImputerMeanFloat32",
        )
//  [python]
def test_imputer_float_inputs(self):
        model = SimpleImputer(missing_values=np.nan, strategy="mean")
        data = [[1, 2], [np.nan, 3], [7, 6]]
        model.fit(data)

        model_onnx = convert_sklearn(
            model, "scikit-learn imputer", [("input", FloatTensorType([None, 2]))]
        )
        self.assertTrue(model_onnx.graph.node is not None)

        # should contain only node
        self.assertEqual(len(model_onnx.graph.node), 1)

        # last node should contain the Imputer
        outputs = model_onnx.graph.output
        self.assertEqual(len(outputs), 1)
        self.assertEqual(outputs[0].type.tensor_type.shape.dim[-1].dim_value, 2)
        dump_data_and_model(
            np.array(data, dtype=np.float32),
            model,
            model_onnx,
            basename="SklearnImputerMeanFloat32",
        )
//  [python]
def _transfer(X):
    imputers = [['Imputer_{i}'.format(i=i), Imputer()] for i in range(19)] 
    imputers[0][1] = imputers[6][1] = imputers[8][1] = imputers[9][1] = imputers[10][1] = imputers[11][1] = imputers[12][1] = imputers[13][1] = imputers[14][1] = imputers[15][1] = imputers[16][1] = imputers[17][1] = Imputer(strategy='most_frequent')
    step1 = ('FeatureUnionExt', FeatureUnionExt(transformer_list=imputers, idx_list=[[i] for i in range(19)]))
    step2 = ('OneHotEncoder', OneHotEncoder(categorical_features=[0, 6, 8, 14, 15, 16, 17], sparse=False))
    step3 = ('StandardScaler', StandardScaler())
    pipeline = PipelineExt(steps=[step1, step2, step3])
    X = pipeline.fit_transform(X)
    return X
//  [python]
def _transfer(X):
    imputers = [['Imputer_{i}'.format(i=i), SimpleImputer()] for i in range(19)]
    imputers[0][1] = imputers[6][1] = imputers[8][1] = imputers[9][1] = imputers[10][1] = imputers[11][1] = imputers[12][1] = imputers[13][1] = imputers[14][1] = imputers[15][1] = imputers[16][1] = imputers[17][1] = SimpleImputer(strategy='most_frequent')
    step1 = ('FeatureUnionExt', FeatureUnionExt(transformer_list=imputers, idx_list=[[i] for i in range(19)]))
    step2 = ('OneHotEncoder', OneHotEncoder(categorical_features=[0, 6, 8, 14, 15, 16, 17], sparse=False))
    step3 = ('StandardScaler', StandardScaler())
    pipeline = PipelineExt(steps=[step1, step2, step3])
    X = pipeline.fit_transform(X)
    return X
//  [python]
get_text_data = FunctionTransformer(combine_text_columns, validate=False)
get_numeric_data = FunctionTransformer(lambda x: x[NUMERIC_COLUMNS], validate=False)

# Create the token pattern: TOKENS_ALPHANUMERIC
TOKENS_ALPHANUMERIC = '[A-Za-z0-9]+(?=\\s+)'

# Instantiate pipeline: pl
pl = Pipeline([
        ('union', FeatureUnion(
            transformer_list = [
                ('numeric_features', Pipeline([
                    ('selector', get_numeric_data),
                    ('imputer', Imputer())
                ])),
                ('text_features', Pipeline([
                    ('selector', get_text_data),
                    ('vectorizer', CountVectorizer(token_pattern=TOKENS_ALPHANUMERIC,
                                                   ngram_range=(1, 2))),
                    ('dim_red', SelectKBest(chi2, chi_k))
                ]))
             ]
        )),
        ('scale', MaxAbsScaler()),
        ('clf', OneVsRestClassifier(LogisticRegression()))
    ])
//  [python]
get_text_data = FunctionTransformer(combine_text_columns, validate=False)
get_numeric_data = FunctionTransformer(lambda x: x[NUMERIC_COLUMNS], validate=False)

# Create the token pattern: TOKENS_ALPHANUMERIC
TOKENS_ALPHANUMERIC = '[A-Za-z0-9]+(?=\\s+)'

# Instantiate pipeline: pl
pl = Pipeline([
        ('union', FeatureUnion(
            transformer_list = [
                ('numeric_features', Pipeline([
                    ('selector', get_numeric_data),
                    ('imputer', SimpleImputer())
                ])),
                ('text_features', Pipeline([
                    ('selector', get_text_data),
                    ('vectorizer', CountVectorizer(token_pattern=TOKENS_ALPHANUMERIC,
                                                   ngram_range=(1, 2))),
                    ('dim_red', SelectKBest(chi2, chi_k))
                ]))
             ]
        )),
        ('scale', MaxAbsScaler()),
        ('clf', OneVsRestClassifier(LogisticRegression()))
    ])
//  [python]
def train_model(titanic_data_path, model_output_path):
    print('Loading the data...')
    try:
        with tf.gfile.Open(titanic_data_path, 'r') as data_file:
            train_df = pd.read_csv(data_file)
            print('Number of samples: {}'.format(train_df.shape[0]))

            target_name = 'Survived'
            feature_names = ['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Embarked']

            print('Preparing the features...')
            train_features = train_df[feature_names].copy()
            train_features['Age'] = Imputer().fit_transform(train_features['Age'].values.reshape(-1, 1))
            embarked = train_features['Embarked']
            train_features['Embarked'] = embarked.fillna(embarked.mode()[0])
            train_features = pd.get_dummies(train_features)
            train_target = train_df[target_name]

            print('Training the model...')
            parameters = {'max_depth': [2, 3, 4, 5, 6, 7], 'n_estimators': [50, 100, 150, 200]}
            gsc = GridSearchCV(GradientBoostingClassifier(), parameters, n_jobs=-1, cv=5)
            gsc.fit(train_features, train_target)
            print('Best Hyper Parameters: {}'.format(gsc.best_params_))
            print('Accuracy: {}'.format(gsc.best_score_))

            with tf.gfile.Open(model_output_path, 'wb') as model_file:
                joblib.dump(gsc.best_estimator_, model_file, protocol=1)
    except Exception as e:
        print('Error: {}'.format(e))
//  [python]
def train_model(titanic_data_path, model_output_path):
    print('Loading the data...')
    try:
        with tf.gfile.Open(titanic_data_path, 'r') as data_file:
            train_df = pd.read_csv(data_file)
            print('Number of samples: {}'.format(train_df.shape[0]))

            target_name = 'Survived'
            feature_names = ['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Embarked']

            print('Preparing the features...')
            train_features = train_df[feature_names].copy()
            train_features['Age'] = SimpleImputer().fit_transform(train_features['Age'].values.reshape(-1, 1))
            embarked = train_features['Embarked']
            train_features['Embarked'] = embarked.fillna(embarked.mode()[0])
            train_features = pd.get_dummies(train_features)
            train_target = train_df[target_name]

            print('Training the model...')
            parameters = {'max_depth': [2, 3, 4, 5, 6, 7], 'n_estimators': [50, 100, 150, 200]}
            gsc = GridSearchCV(GradientBoostingClassifier(), parameters, n_jobs=-1, cv=5)
            gsc.fit(train_features, train_target)
            print('Best Hyper Parameters: {}'.format(gsc.best_params_))
            print('Accuracy: {}'.format(gsc.best_score_))

            with tf.gfile.Open(model_output_path, 'wb') as model_file:
                joblib.dump(gsc.best_estimator_, model_file, protocol=1)
    except Exception as e:
        print('Error: {}'.format(e))
//  [python]
def test_calibration_nan_imputer():
    """Test that calibration can accept nan"""
    X, y = make_classification(n_samples=10, n_features=2,
                               n_informative=2, n_redundant=0,
                               random_state=42)
    X[0, 0] = np.nan
    clf = Pipeline(
        [('imputer', Imputer()),
         ('rf', RandomForestClassifier(n_estimators=1))])
    clf_c = CalibratedClassifierCV(clf, cv=2, method='isotonic')
    clf_c.fit(X, y)
    clf_c.predict(X)
//  [python]
def test_calibration_nan_imputer():
    """Test that calibration can accept nan"""
    X, y = make_classification(n_samples=10, n_features=2,
                               n_informative=2, n_redundant=0,
                               random_state=42)
    X[0, 0] = np.nan
    clf = Pipeline(
        [('imputer', SimpleImputer()),
         ('rf', RandomForestClassifier(n_estimators=1))])
    clf_c = CalibratedClassifierCV(clf, cv=2, method='isotonic')
    clf_c.fit(X, y)
    clf_c.predict(X)
//  [python]
def __init__(self):
        self.reg = Pipeline([
            ('imputer', Imputer(strategy='median')),
            ('regressor', linear_model.LinearRegression())
        ])
//  [python]
def __init__(self):
        self.reg = Pipeline([
            ('imputer', SimpleImputer(strategy='median')),
            ('regressor', linear_model.LinearRegression())
        ])
//  [python]
def fit(self, X, y=None):
        self.imp = Imputer(strategy=self.strategy)
        self.imp.fit(X)
        self.statistics_ = pd.Series(self.imp.statistics_, index=X.columns)
        return self
//  [python]
def fit(self, X, y=None):
        self.imp = SimpleImputer(strategy=self.strategy)
        self.imp.fit(X)
        self.statistics_ = pd.Series(self.imp.statistics_, index=X.columns)
        return self
//  [python]
def test_conversion_boston(self):

        from sklearn.datasets import load_boston

        scikit_data = load_boston()

        sh = scikit_data.data.shape

        rn.seed(0)
        missing_value_indices = [
            (rn.randint(sh[0]), rn.randint(sh[1])) for k in range(sh[0])
        ]

        for strategy in ["mean", "median", "most_frequent"]:
            for missing_value in [0, "NaN", -999]:
                # SimpleImputer >=0.22 does not accept missing values encoded as NaN.
                if _SKLEARN_VERSION >= StrictVersion("0.22"):
                    if missing_value == "NaN":
                        continue

                X = np.array(scikit_data.data).copy()

                for i, j in missing_value_indices:
                    X[i, j] = missing_value

                model = Imputer(missing_values=missing_value, strategy=strategy)
                model = model.fit(X)

                tr_X = model.transform(X.copy())

                spec = converter.convert(model, scikit_data.feature_names, "out")

                input_data = [dict(zip(scikit_data.feature_names, row)) for row in X]

                output_data = [{"out": row} for row in tr_X]

                result = evaluate_transformer(spec, input_data, output_data)

                assert result["num_errors"] == 0
//  [python]
def test_conversion_boston(self):

        from sklearn.datasets import load_boston
        from sklearn.impute import SimpleImputer

        scikit_data = load_boston()

        sh = scikit_data.data.shape

        rn.seed(0)
        missing_value_indices = [
            (rn.randint(sh[0]), rn.randint(sh[1])) for k in range(sh[0])
        ]

        for strategy in ["mean", "median", "most_frequent"]:
            for missing_value in [0, "NaN", -999]:
                # SimpleImputer >=0.22 does not accept missing values encoded as NaN.
                if _SKLEARN_VERSION >= StrictVersion("0.22"):
                    if missing_value == "NaN":
                        continue

                X = np.array(scikit_data.data).copy()

                for i, j in missing_value_indices:
                    X[i, j] = missing_value

                model = SimpleImputer(missing_values=missing_value, strategy=strategy)
                model = model.fit(X)

                tr_X = model.transform(X.copy())

                spec = converter.convert(model, scikit_data.feature_names, "out")

                input_data = [dict(zip(scikit_data.feature_names, row)) for row in X]

                output_data = [{"out": row} for row in tr_X]

                result = evaluate_transformer(spec, input_data, output_data)

                assert result["num_errors"] == 0
//  [python]
def load_train_fs2():
    train_data = pd.read_csv('./train_v2.csv')
    c['f137'] = train_data.f137.replace('NA', np.nan)
    imp = Imputer(missing_values=np.nan, strategy='mean', axis=0)
    train_data = imp.fit_transform(train_data)
    return train_data.astype(float)
//  [python]
def load_train_fs2():
    train_data = pd.read_csv('./train_v2.csv')
    c['f137'] = train_data.f137.replace('NA', np.nan)
    imp = SimpleImputer(missing_values=np.nan, strategy='mean')
    train_data = imp.fit_transform(train_data)
    return train_data.astype(float)
//  [python]
def load_test_fs2():
    test_x = pd.read_csv('./test_v2.csv')
    test_x['f137'] = test_x.f137.replace('NA', np.nan)
    test_x['f276'] = test_x.f137.replace('NA', np.nan)
    imp = Imputer(missing_values=np.nan, strategy='mean', axis=0)
    test_x = imp.fit_transform(test_x)
    return test_x.astype(float)
//  [python]
def load_test_fs2():
    test_x = pd.read_csv('./test_v2.csv')
    test_x['f137'] = test_x.f137.replace('NA', np.nan)
    test_x['f276'] = test_x.f137.replace('NA', np.nan)
    imp = SimpleImputer(missing_values=np.nan, strategy='mean')
    test_x = imp.fit_transform(test_x)
    return test_x.astype(float)
//  [python]
def remove_nan(x):
        """remove NaN values from data vectors"""
        imp = Imputer(missing_values='NaN', strategy='mean', axis=0)
        x_clean = imp.fit_transform(x)
        return x_clean
//  [python]
def remove_nan(x):
        """remove NaN values from data vectors"""
        imp = SimpleImputer(missing_values='NaN', strategy='mean')
        x_clean = imp.fit_transform(x)
        return x_clean
//  [python]
def pre_process(raw_data, categorical, impute=True, standardize=True, one_hot_encode=True):
    """
    Pre-process one dataset.

    Args:
        raw_data (np.ndarray):    raw features of the n-by-d dataset, without indices and headings.
        categorical (list):       a boolean list of length d indicating whether each raw feature is categorical.
        impute (bool):            whether to impute missing entries or not.
        standardize (bool):       whether to standardize each feature or not.
        one_hot_encode (bool):    whether to use one hot encoding to pre-process categorical features or not.
    Returns:
        np.ndarray:               pre-processed dataset.
    """
    # list of pre-processed arrays (sub-portions of dataset)
    processed = []

    # whether to impute missing entries
    if impute:
        # if there are any categorical features
        if np.array(categorical).any():
            raw_categorical = raw_data[:, categorical]
            # impute missing entries in categorical features using the most frequent number
            imp_categorical = Imputer(missing_values='NaN', strategy='most_frequent', axis=0, copy=False)
            processed.append(imp_categorical.fit_transform(raw_categorical))

        # if there are any numeric features
        if np.invert(categorical).any():
            raw_numeric = raw_data[:, np.invert(categorical)]
            # impute missing entries in non-categorical features using mean
            imp_numeric = Imputer(missing_values='NaN', strategy='mean', axis=0, copy=False)
            processed.append(imp_numeric.fit_transform(raw_numeric))

        # data has now been re-ordered so all categorical features appear first
        categorical = np.array(sorted(categorical, reverse=True))
        processed_data = np.hstack(tuple(processed))

    else:
        processed_data = raw_data

    # one-hot encoding for categorical features (only if there exist any)
    if one_hot_encode and np.array(categorical).any():
        encoder = OneHotEncoder(categorical_features=categorical)
        processed_data = encoder.fit_transform(processed_data).toarray()
        categorical = np.zeros(processed_data.shape[1], dtype=bool)
            
    # standardize all numeric and one-hot encoded categorical features
    if standardize:
        processed_data[:, np.invert(categorical)] = scale(processed_data[:, np.invert(categorical)])
        
    print('Data pre-processing finished')
    return processed_data, categorical
//  [python]
def pre_process(raw_data, categorical, impute=True, standardize=True, one_hot_encode=True):
    """
    Pre-process one dataset.

    Args:
        raw_data (np.ndarray):    raw features of the n-by-d dataset, without indices and headings.
        categorical (list):       a boolean list of length d indicating whether each raw feature is categorical.
        impute (bool):            whether to impute missing entries or not.
        standardize (bool):       whether to standardize each feature or not.
        one_hot_encode (bool):    whether to use one hot encoding to pre-process categorical features or not.
    Returns:
        np.ndarray:               pre-processed dataset.
    """
    # list of pre-processed arrays (sub-portions of dataset)
    processed = []

    # whether to impute missing entries
    if impute:
        # if there are any categorical features
        if np.array(categorical).any():
            raw_categorical = raw_data[:, categorical]
            # impute missing entries in categorical features using the most frequent number
            imp_categorical = SimpleImputer(missing_values=np.nan, strategy='most_frequent')
            processed.append(imp_categorical.fit_transform(raw_categorical))

        # if there are any numeric features
        if np.invert(categorical).any():
            raw_numeric = raw_data[:, np.invert(categorical)]
            # impute missing entries in non-categorical features using mean
            imp_numeric = SimpleImputer(missing_values=np.nan, strategy='mean')
            processed.append(imp_numeric.fit_transform(raw_numeric))

        # data has now been re-ordered so all categorical features appear first
        categorical = np.array(sorted(categorical, reverse=True))
        processed_data = np.hstack(tuple(processed))

    else:
        processed_data = raw_data

    # one-hot encoding for categorical features (only if there exist any)
    if one_hot_encode and np.array(categorical).any():
        encoder = OneHotEncoder(categorical_features=categorical, handle_unknown='ignore')
        processed_data = encoder.fit_transform(processed_data).toarray()
        categorical = np.zeros(processed_data.shape[1], dtype=bool)
            
    # standardize all numeric and one-hot encoded categorical features
    if standardize:
        processed_data[:, np.invert(categorical)] = scale(processed_data[:, np.invert(categorical)])
        
    print('Data pre-processing finished')
    return processed_data, categorical
//  [python]
def read_input(file_name, test_size=0.25):
  """Read input data and split it into train and test."""
  data = pd.read_csv(file_name[0])
  data.dropna(axis=0, subset=['SalePrice'], inplace=True)

  y = data.SalePrice
  X = data.drop(['SalePrice'], axis=1).select_dtypes(exclude=['object'])

  train_X, test_X, train_y, test_y = train_test_split(X.values,
                                                      y.values,
                                                      test_size=test_size,
                                                      shuffle=False)

  imputer = Imputer()
  train_X = imputer.fit_transform(train_X)
  test_X = imputer.transform(test_X)

  return (train_X, train_y), (test_X, test_y)
//  [python]
def read_input(file_name, test_size=0.25):
  """Read input data and split it into train and test."""
  data = pd.read_csv(file_name[0])
  data.dropna(axis=0, subset=['SalePrice'], inplace=True)

  y = data.SalePrice
  X = data.drop(['SalePrice'], axis=1).select_dtypes(exclude=['object'])

  train_X, test_X, train_y, test_y = train_test_split(X.values,
                                                      y.values,
                                                      test_size=test_size,
                                                      shuffle=False)

  imputer = SimpleImputer()
  train_X = imputer.fit_transform(train_X)
  test_X = imputer.transform(test_X)

  return (train_X, train_y), (test_X, test_y)
//  [python]
def impute_data(self,x):
        """Imputes data set containing Nan values"""
        imp = Imputer(missing_values='NaN', strategy='mean', axis=0)
        return imp.fit_transform(x)
//  [python]
def impute_data(self,x):
        """Imputes data set containing Nan values"""
        imp = SimpleImputer(missing_values=np.nan, strategy='mean')
        return imp.fit_transform(x)
//  [python]
def imputer(args):
	#https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.Imputer.html
	#TODO: deprecated

	return Imputer(missing_values='NaN', strategy=args['strategy'], axis=0, copy=True)
//  [python]
def imputer(args):
	from sklearn.impute import SimpleImputer
	return SimpleImputer(missing_values='NaN', strategy=args['strategy'])
//  [python]
X_train, X_test, y_train, y_test = train_test_split(sample_df[['numeric', 'with_missing']],pd.get_dummies(sample_df['label']),random_state=456)

# Insantiate Pipeline object: pl
pl = Pipeline([('imp', Imputer()),   ('clf',OneVsRestClassifier(LogisticRegression()))]
)
//  [python]
X_train, X_test, y_train, y_test = train_test_split(sample_df[['numeric', 'with_missing']],pd.get_dummies(sample_df['label']),random_state=456)

# Insantiate Pipeline object: pl
pl = Pipeline([('imp', SimpleImputer()),   ('clf',OneVsRestClassifier(LogisticRegression()))]
)
//  [python]
def test_imputer(sdf, pdf):
    hdf = sdf.toHandy()
    hdf_filled = hdf.stratify(['Pclass']).fill(continuous=['Age'])
    himputer = hdf_filled.transformers.imputer()

    sdf_filled = himputer.transform(sdf)
    sage = sdf_filled.sort('PassengerId').toHandy().cols['Age'][:].values

    pdf_filled = []
    for pclass in [1, 2, 3]:
        filtered = pdf.query('Pclass == {}'.format(pclass))[['PassengerId', 'Age']]
        imputer = Imputer(strategy='mean').fit(filtered)
        pdf_filled.append(imputer.transform(filtered))
    pdf_filled = sorted(np.concatenate(pdf_filled, axis=0), key=itemgetter(0))
    age = list(map(itemgetter(1), pdf_filled))

    npt.assert_array_equal(sage, age)
//  [python]
def test_imputer(sdf, pdf):
    hdf = sdf.toHandy()
    hdf_filled = hdf.stratify(['Pclass']).fill(continuous=['Age'])
    himputer = hdf_filled.transformers.imputer()

    sdf_filled = himputer.transform(sdf)
    sage = sdf_filled.sort('PassengerId').toHandy().cols['Age'][:].values

    pdf_filled = []
    for pclass in [1, 2, 3]:
        filtered = pdf.query('Pclass == {}'.format(pclass))[['PassengerId', 'Age']]
        imputer = SimpleImputer(strategy='mean').fit(filtered)
        pdf_filled.append(imputer.transform(filtered))
    pdf_filled = sorted(np.concatenate(pdf_filled, axis=0), key=itemgetter(0))
    age = list(map(itemgetter(1), pdf_filled))

    npt.assert_array_equal(sage, age)
//  [python]
def test_run_model_on_task(self):
        task = openml.tasks.get_task(1)  # anneal; crossvalidation
        # using most_frequent imputer since dataset has mixed types and to keep things simple
        pipe = sklearn.pipeline.Pipeline(
            [
                ("imp", SimpleImputer(strategy="most_frequent")),
                ("dummy", sklearn.dummy.DummyClassifier()),
            ],
        )
        openml.runs.run_model_on_task(pipe, task, dataset_format="array")
//  [python]
def test_run_model_on_task(self):
        task = openml.tasks.get_task(1)  # anneal; crossvalidation
        # using most_frequent imputer since dataset has mixed types and to keep things simple
        pipe = sklearn.pipeline.Pipeline(
            [
                ("imp", sklearn.impute.SimpleImputer(strategy="most_frequent")),
                ("dummy", sklearn.dummy.DummyClassifier()),
            ],
        )
        openml.runs.run_model_on_task(pipe, task, dataset_format="array")
//  [python]
def test_deserialize_with_defaults(self):
        # used the 'initialize_with_defaults' flag of the deserialization
        # method to return a flow that contains default hyperparameter
        # settings.
        steps = [
            ("Imputer", SimpleImputer()),
            ("OneHotEncoder", sklearn.preprocessing.OneHotEncoder()),
            ("Estimator", sklearn.tree.DecisionTreeClassifier()),
        ]
        pipe_orig = sklearn.pipeline.Pipeline(steps=steps)

        pipe_adjusted = sklearn.clone(pipe_orig)
        if LooseVersion(sklearn.__version__) < "0.23":
            params = {
                "Imputer__strategy": "median",
                "OneHotEncoder__sparse": False,
                "Estimator__min_samples_leaf": 42,
            }
        else:
            params = {
                "Imputer__strategy": "mean",
                "OneHotEncoder__sparse": True,
                "Estimator__min_samples_leaf": 1,
            }
        pipe_adjusted.set_params(**params)
        flow = self.extension.model_to_flow(pipe_adjusted)
        pipe_deserialized = self.extension.flow_to_model(flow, initialize_with_defaults=True)

        # we want to compare pipe_deserialized and pipe_orig. We use the flow
        # equals function for this
        assert_flows_equal(
            self.extension.model_to_flow(pipe_orig),
            self.extension.model_to_flow(pipe_deserialized),
        )
//  [python]
def test_deserialize_with_defaults(self):
        # used the 'initialize_with_defaults' flag of the deserialization
        # method to return a flow that contains default hyperparameter
        # settings.
        steps = [
            ("Imputer", sklearn.impute.SimpleImputer()),
            ("OneHotEncoder", sklearn.preprocessing.OneHotEncoder()),
            ("Estimator", sklearn.tree.DecisionTreeClassifier()),
        ]
        pipe_orig = sklearn.pipeline.Pipeline(steps=steps)

        pipe_adjusted = sklearn.clone(pipe_orig)
        if LooseVersion(sklearn.__version__) < "0.23":
            params = {
                "Imputer__strategy": "median",
                "OneHotEncoder__sparse": False,
                "Estimator__min_samples_leaf": 42,
            }
        else:
            params = {
                "Imputer__strategy": "mean",
                "OneHotEncoder__sparse": True,
                "Estimator__min_samples_leaf": 1,
            }
        pipe_adjusted.set_params(**params)
        flow = self.extension.model_to_flow(pipe_adjusted)
        pipe_deserialized = self.extension.flow_to_model(flow, initialize_with_defaults=True)

        # we want to compare pipe_deserialized and pipe_orig. We use the flow
        # equals function for this
        assert_flows_equal(
            self.extension.model_to_flow(pipe_orig),
            self.extension.model_to_flow(pipe_deserialized),
        )
//  [python]
def test_deserialize_adaboost_with_defaults(self):
        # used the 'initialize_with_defaults' flag of the deserialization
        # method to return a flow that contains default hyperparameter
        # settings.
        steps = [
            ("Imputer", SimpleImputer()),
            ("OneHotEncoder", sklearn.preprocessing.OneHotEncoder()),
            (
                "Estimator",
                sklearn.ensemble.AdaBoostClassifier(sklearn.tree.DecisionTreeClassifier()),
            ),
        ]
        pipe_orig = sklearn.pipeline.Pipeline(steps=steps)

        pipe_adjusted = sklearn.clone(pipe_orig)
        if LooseVersion(sklearn.__version__) < "0.22":
            params = {
                "Imputer__strategy": "median",
                "OneHotEncoder__sparse": False,
                "Estimator__n_estimators": 10,
            }
        else:
            params = {
                "Imputer__strategy": "mean",
                "OneHotEncoder__sparse": True,
                "Estimator__n_estimators": 50,
            }
        pipe_adjusted.set_params(**params)
        flow = self.extension.model_to_flow(pipe_adjusted)
        pipe_deserialized = self.extension.flow_to_model(flow, initialize_with_defaults=True)

        # we want to compare pipe_deserialized and pipe_orig. We use the flow
        # equals function for this
        assert_flows_equal(
            self.extension.model_to_flow(pipe_orig),
            self.extension.model_to_flow(pipe_deserialized),
        )
//  [python]
def test_deserialize_complex_with_defaults(self):
        # used the 'initialize_with_defaults' flag of the deserialization
        # method to return a flow that contains default hyperparameter
        # settings.
        steps = [
            ("Imputer", SimpleImputer()),
            ("OneHotEncoder", sklearn.preprocessing.OneHotEncoder()),
            (
                "Estimator",
                sklearn.ensemble.AdaBoostClassifier(
                    sklearn.ensemble.BaggingClassifier(
                        sklearn.ensemble.GradientBoostingClassifier(),
                    ),
                ),
            ),
        ]
        pipe_orig = sklearn.pipeline.Pipeline(steps=steps)

        pipe_adjusted = sklearn.clone(pipe_orig)
        impute_strategy = "median" if LooseVersion(sklearn.__version__) < "0.23" else "mean"
        sparse = LooseVersion(sklearn.__version__) >= "0.23"
        estimator_name = (
            "base_estimator" if LooseVersion(sklearn.__version__) < "1.2" else "estimator"
        )
        params = {
            "Imputer__strategy": impute_strategy,
            "OneHotEncoder__sparse": sparse,
            "Estimator__n_estimators": 10,
            f"Estimator__{estimator_name}__n_estimators": 10,
            f"Estimator__{estimator_name}__{estimator_name}__learning_rate": 0.1,
        }

        pipe_adjusted.set_params(**params)
        flow = self.extension.model_to_flow(pipe_adjusted)
        pipe_deserialized = self.extension.flow_to_model(flow, initialize_with_defaults=True)

        # we want to compare pipe_deserialized and pipe_orig. We use the flow
        # equals function for this
        assert_flows_equal(
            self.extension.model_to_flow(pipe_orig),
            self.extension.model_to_flow(pipe_deserialized),
        )
//  [python]
def test_run_model_on_fold_clustering(self):
        # There aren't any regression tasks on the test server
        openml.config.server = self.production_server
        task = openml.tasks.get_task(126033)

        X = task.get_X(dataset_format="array")

        pipeline = sklearn.pipeline.Pipeline(
            steps=[("imp", SimpleImputer()), ("clf", sklearn.cluster.KMeans())],
        )
        # TODO add some mocking here to actually test the innards of this function, too!
        res = self.extension._run_model_on_fold(
            model=pipeline,
            task=task,
            fold_no=0,
            rep_no=0,
            X_train=X,
        )

        y_hat, y_hat_proba, user_defined_measures, trace = res

        # predictions
        assert isinstance(y_hat, np.ndarray)
        assert y_hat.shape == (X.shape[0],)
        assert y_hat_proba is None

        # check user defined measures
        fold_evaluations: dict[str, dict[int, dict[int, float]]]  = collections.defaultdict(lambda: collections.defaultdict(dict))
        for measure in user_defined_measures:
            fold_evaluations[measure][0][0] = user_defined_measures[measure]

        # trace. SGD does not produce any
        assert trace is None

        self._check_fold_timing_evaluations(
            fold_evaluations,
            num_repeats=1,
            num_folds=1,
            task_type=task.task_type_id,
            check_scores=False,
        )
//  [python]
def test_run_model_on_fold_clustering(self):
        # There aren't any regression tasks on the test server
        openml.config.server = self.production_server
        task = openml.tasks.get_task(126033)

        X = task.get_X(dataset_format="array")

        pipeline = sklearn.pipeline.Pipeline(
            steps=[("imp", sklearn.impute.SimpleImputer()), ("clf", sklearn.cluster.KMeans())],
        )
        # TODO add some mocking here to actually test the innards of this function, too!
        res = self.extension._run_model_on_fold(
            model=pipeline,
            task=task,
            fold_no=0,
            rep_no=0,
            X_train=X,
        )

        y_hat, y_hat_proba, user_defined_measures, trace = res

        # predictions
        assert isinstance(y_hat, np.ndarray)
        assert y_hat.shape == (X.shape[0],)
        assert y_hat_proba is None

        # check user defined measures
        fold_evaluations: dict[str, dict[int, dict[int, float]]]  = collections.defaultdict(lambda: collections.defaultdict(dict))
        for measure in user_defined_measures:
            fold_evaluations[measure][0][0] = user_defined_measures[measure]

        # trace. SGD does not produce any
        assert trace is None

        self._check_fold_timing_evaluations(
            fold_evaluations,
            num_repeats=1,
            num_folds=1,
            task_type=task.task_type_id,
            check_scores=False,
        )
//  [python]
def test_run_model_on_fold_classification_1_array(self):
        task = openml.tasks.get_task(1)  # anneal; crossvalidation

        X, y = task.get_X_and_y()
        train_indices, test_indices = task.get_train_test_split_indices(repeat=0, fold=0, sample=0)
        X_train = X[train_indices]
        y_train = y[train_indices]
        X_test = X[test_indices]
        y_test = y[test_indices]

        pipeline = sklearn.pipeline.Pipeline(
            steps=[("imp", SimpleImputer()), ("clf", sklearn.tree.DecisionTreeClassifier())],
        )
        # TODO add some mocking here to actually test the innards of this function, too!
        res = self.extension._run_model_on_fold(
            model=pipeline,
            task=task,
            fold_no=0,
            rep_no=0,
            X_train=X_train,
            y_train=y_train,
            X_test=X_test,
        )

        y_hat, y_hat_proba, user_defined_measures, trace = res

        # predictions
        assert isinstance(y_hat, np.ndarray)
        assert y_hat.shape == y_test.shape
        assert isinstance(y_hat_proba, pd.DataFrame)
        assert y_hat_proba.shape == (y_test.shape[0], 6)
        np.testing.assert_array_almost_equal(np.sum(y_hat_proba, axis=1), np.ones(y_test.shape))
        # The class '4' (at index 3) is not present in the training data. We check that the
        # predicted probabilities for that class are zero!
        np.testing.assert_array_almost_equal(
            y_hat_proba.iloc[:, 3].to_numpy(),
            np.zeros(y_test.shape),
        )
        for i in (0, 1, 2, 4, 5):
            assert np.any(y_hat_proba.iloc[:, i].to_numpy() != np.zeros(y_test.shape))

        # check user defined measures
        fold_evaluations: dict[str, dict[int, dict[int, float]]] = collections.defaultdict(lambda: collections.defaultdict(dict))
        for measure in user_defined_measures:
            fold_evaluations[measure][0][0] = user_defined_measures[measure]

        # trace. SGD does not produce any
        assert trace is None

        self._check_fold_timing_evaluations(
            fold_evaluations,
            num_repeats=1,
            num_folds=1,
            task_type=task.task_type_id,
            check_scores=False,
        )
//  [python]
def test_run_model_on_fold_regression(self):
        # There aren't any regression tasks on the test server
        openml.config.server = self.production_server
        task = openml.tasks.get_task(2999)

        X, y = task.get_X_and_y()
        train_indices, test_indices = task.get_train_test_split_indices(repeat=0, fold=0, sample=0)
        X_train = X[train_indices]
        y_train = y[train_indices]
        X_test = X[test_indices]
        y_test = y[test_indices]

        pipeline = sklearn.pipeline.Pipeline(
            steps=[("imp", SimpleImputer()), ("clf", sklearn.tree.DecisionTreeRegressor())],
        )
        # TODO add some mocking here to actually test the innards of this function, too!
        res = self.extension._run_model_on_fold(
            model=pipeline,
            task=task,
            fold_no=0,
            rep_no=0,
            X_train=X_train,
            y_train=y_train,
            X_test=X_test,
        )

        y_hat, y_hat_proba, user_defined_measures, trace = res

        # predictions
        assert isinstance(y_hat, np.ndarray)
        assert y_hat.shape == y_test.shape
        assert y_hat_proba is None

        # check user defined measures
        fold_evaluations: dict[str, dict[int, dict[int, float]]]  = collections.defaultdict(lambda: collections.defaultdict(dict))
        for measure in user_defined_measures:
            fold_evaluations[measure][0][0] = user_defined_measures[measure]

        # trace. SGD does not produce any
        assert trace is None

        self._check_fold_timing_evaluations(
            fold_evaluations,
            num_repeats=1,
            num_folds=1,
            task_type=task.task_type_id,
            check_scores=False,
        )