jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def build_estimator(config):
    """
    Build the estimator object from the parameters in config.

    :param config:
    :return:
    """
    regressor = config['Model']['regressor']
    sklearn_kwargs = config['Model']['Parameters']
    train_individual = config['Model']['train_individual']
    ada_boost = config['Model'].get('Ada boosting', None)
    rain_tuning = config['Model'].get('Rain tuning', None)
    bootstrap = config['Model'].get('Bootstrapping', None)
    Regressor = get_object('sklearn.%s' % regressor)
    if config['verbose']:
        print('build_estimator: using sklearn.%s as estimator...' % regressor)

    from sklearn.impute import SimpleImputer
    from sklearn.preprocessing import StandardScaler as Scaler
    from sklearn.pipeline import Pipeline

    # Create and train the learning algorithm
    if config['verbose']:
        print('build_estimator: here are the parameters passed to the learning algorithm...')
        print(sklearn_kwargs)

    # Create the pipeline list
    pipeline = [("imputer", SimpleImputer(missing_values=np.nan, strategy="mean", axis=0))]
    if config['Model']['predict_timeseries']:
        pipeline_timeseries = [("imputer", SimpleImputer(missing_values=np.nan, strategy="mean", axis=0))]

    if not (regressor.startswith('ensemble')):
        # Need to add feature scaling
        pipeline.append(("scaler", Scaler()))
        if config['Model']['predict_timeseries']:
            pipeline_timeseries.append(("scaler", Scaler()))

    # Create the regressor object
    regressor_obj = Regressor(**sklearn_kwargs)
    if ada_boost is not None:
        if config['verbose']:
            print('build_estimator: using Ada boosting...')
        from sklearn.ensemble import AdaBoostRegressor
        regressor_obj = AdaBoostRegressor(regressor_obj, **ada_boost)
    if train_individual:
        if config['verbose']:
            print('build_estimator: training separate models for each parameter...')
        from sklearn.multioutput import MultiOutputRegressor
        multi_regressor = MultiOutputRegressor(regressor_obj, 4)
        pipeline.append(("regressor", multi_regressor))
    else:
        pipeline.append(("regressor", regressor_obj))
    if config['Model']['predict_timeseries']:
        pipeline_timeseries.append(("regressor", regressor_obj))

    # Make the final estimator with a Pipeline
    if config['Model']['predict_timeseries']:
        estimator = TimeSeriesEstimator(Pipeline(pipeline), Pipeline(pipeline_timeseries))
    else:
        estimator = Pipeline(pipeline)

    if rain_tuning is not None and regressor.startswith('ensemble'):
        if config['verbose']:
            print('build_estimator: using rain tuning...')
        rain_kwargs = rain_tuning.copy()
        rain_kwargs.pop('use_raw_rain', None)
        estimator = RainTuningEstimator(estimator, **rain_kwargs)

    # Add bootstrapping if requested
    if bootstrap is not None:
        if config['verbose']:
            print('build_estimator: using bootstrapping ensemble...')
        estimator = BootStrapEnsembleEstimator(estimator, **bootstrap)

    return estimator
//  [python]
def MKL_training_single(Training,label,MKL_param):

    feature_name = Training.columns.tolist()
    label=label.astype(np.float64).values
    
    train_psi = [feature_name.index(x) for x in feature_name if x.startswith("ENSG")]
    train_exp = [feature_name.index(x) for x in feature_name if not x.startswith("ENSG")]
    kernel_dict = {'GaussianKernel':{'1':train_psi,'2':train_exp},'PolyKernel':{'1':train_psi,'2':train_exp},'LinearKernel':{'1':train_psi,'2':train_exp}}
    
    MKL_pipe = Pipeline([('imr',Imputer(missing_values='NaN',strategy='mean',axis=0)),
                         ('scl',StandardScaler()),  
                         ('mkl',mkl_classifier(kernel_dict=kernel_dict))])
   
    MKL_pipe.set_params(**MKL_param)
    
    MKL_pipe.fit(Training,label)
    
    return(MKL_pipe)
//  [python]
def MKL_training_single(Training,label,MKL_param):

    feature_name = Training.columns.tolist()
    label=label.astype(np.float64).values
    
    train_psi = [feature_name.index(x) for x in feature_name if x.startswith("ENSG")]
    train_exp = [feature_name.index(x) for x in feature_name if not x.startswith("ENSG")]
    kernel_dict = {'GaussianKernel':{'1':train_psi,'2':train_exp},'PolyKernel':{'1':train_psi,'2':train_exp},'LinearKernel':{'1':train_psi,'2':train_exp}}
    
    MKL_pipe = Pipeline([('imr',SimpleImputer(missing_values='NaN',strategy='mean')),
                         ('scl',StandardScaler()),  
                         ('mkl',mkl_classifier(kernel_dict=kernel_dict))])
   
    MKL_pipe.set_params(**MKL_param)
    
    MKL_pipe.fit(Training,label)
    
    return(MKL_pipe)
//  [python]
def MKL_training(Training,label,MKL_param,cv,threads=6):

    feature_name = Training.columns.tolist()
    label=label.astype(np.float64).values
    
    train_psi = [feature_name.index(x) for x in feature_name if x.startswith("ENSG")]
    train_exp = [feature_name.index(x) for x in feature_name if not x.startswith("ENSG")]
    kernel_dict = {'GaussianKernel':{'1':train_psi,'2':train_exp},'PolyKernel':{'1':train_psi,'2':train_exp},'LinearKernel':{'1':train_psi,'2':train_exp}}
    
    MKL_pipe = Pipeline([('imr',Imputer(missing_values='NaN',strategy='mean',axis=0)),
                         ('scl',StandardScaler()),  
                         ('mkl',mkl_classifier(kernel_dict=kernel_dict))])
    
    MKL_gs = GridSearchCV(estimator=MKL_pipe,
                 param_grid=MKL_param,
                 cv=cv,
                 n_jobs=threads)

    MKL_gs.fit(Training,label)
   
    MKL_pipe.set_params(**MKL_gs.best_params_)
    
    MKL_pipe.fit(Training,label)
    
    return(MKL_pipe)
//  [python]
def MKL_training(Training,label,MKL_param,cv,threads=6):

    feature_name = Training.columns.tolist()
    label=label.astype(np.float64).values
    
    train_psi = [feature_name.index(x) for x in feature_name if x.startswith("ENSG")]
    train_exp = [feature_name.index(x) for x in feature_name if not x.startswith("ENSG")]
    kernel_dict = {'GaussianKernel':{'1':train_psi,'2':train_exp},'PolyKernel':{'1':train_psi,'2':train_exp},'LinearKernel':{'1':train_psi,'2':train_exp}}
    
    MKL_pipe = Pipeline([('imr',SimpleImputer(missing_values='NaN',strategy='mean')),
                         ('scl',StandardScaler()),  
                         ('mkl',mkl_classifier(kernel_dict=kernel_dict))])
    
    MKL_gs = GridSearchCV(estimator=MKL_pipe,
                 param_grid=MKL_param,
                 cv=cv,
                 n_jobs=threads)

    MKL_gs.fit(Training,label)
   
    MKL_pipe.set_params(**MKL_gs.best_params_)
    
    MKL_pipe.fit(Training,label)
    
    return(MKL_pipe)
//  [python]
def impute_and_scale_array(mat, scaling=None):
    """ Impute missing values with mean and scale data included in numpy array.

        Parameters
        ----------
        mat : numpy array
            Array to scale
        scaling : string
            String describing type of scaling to apply.
            Options recognized: 'maxabs', 'minmax', 'std'.
            'maxabs' : scales data to range [-1 to 1].
            'minmax' : scales data to range [-1 to 1].
            'std'    : scales data to normal variable with mean 0 and standard deviation 1.
            (Default: None, no scaling).

        Return
        ----------
        Returns the numpy array imputed with the mean value of the \
        column and scaled by the method specified. If no scaling method is specified, \
        it returns the imputed numpy array.
    """

    # imputer = Imputer(strategy='mean', axis=0, copy=False)
    # imputer = SimpleImputer(strategy='mean', copy=False)
    # Next line is from conditional import. axis=0 is default
    # in old version so it is not necessary.
    imputer = Imputer(strategy="mean", copy=False)
    imputer.fit_transform(mat)

    return scale_array(mat, scaling)
//  [python]
def impute_and_scale_array(mat, scaling=None):
    """ Impute missing values with mean and scale data included in numpy array.

        Parameters
        ----------
        mat : numpy array
            Array to scale
        scaling : string
            String describing type of scaling to apply.
            Options recognized: 'maxabs', 'minmax', 'std'.
            'maxabs' : scales data to range [-1 to 1].
            'minmax' : scales data to range [-1 to 1].
            'std'    : scales data to normal variable with mean 0 and standard deviation 1.
            (Default: None, no scaling).

        Return
        ----------
        Returns the numpy array imputed with the mean value of the \
        column and scaled by the method specified. If no scaling method is specified, \
        it returns the imputed numpy array.
    """

    imputer = SimpleImputer(strategy="mean", copy=False)
    imputer.fit_transform(mat)

    return scale_array(mat, scaling)
//  [python]
def impute(df,kind):
    if kind == 'median':
        imputer = Imputer(missing_values=np.nan,strategy='median',axis=0)
        df[:] = imputer.fit_transform(df)
    return df
//  [python]
def impute(df,kind):
    if kind == 'median':
        imputer = SimpleImputer(missing_values=np.nan,strategy='median')
        df[:] = imputer.fit_transform(df)
    return df
//  [python]
def imputer(args):

	"""Uses scikit-learn's Imputer, imputation transformer for completing missing values.
                
    Parameters
    ----------

    strategy : string
	    The imputation strategy.
    	    If “mean”, then replace missing values using the mean along the axis.
        	If “median”, then replace missing values using the median along the axis.
        	If “most_frequent”, then replace missing using the most frequent value along the axis.

    """
	
	strat = args[1]

	return Imputer(missing_values='NaN', strategy=strat, axis=0, verbose=0, copy=True)
//  [python]
def imputer(args):

	"""Uses scikit-learn's Imputer, imputation transformer for completing missing values.
                
    Parameters
    ----------

    strategy : string
	    The imputation strategy.
    	    If “mean”, then replace missing values using the mean along the axis.
        	If “median”, then replace missing values using the median along the axis.
        	If “most_frequent”, then replace missing using the most frequent value along the axis.

    """
	
	strat = args[1]

	return SimpleImputer(missing_values='NaN', strategy=strat)
//  [python]
def impute_X(X):
    # The X_obs matrix (molecular descriptors) still has NaN values that
    # need to be imputed.  
    imputer = Imputer(missing_values=np.nan,strategy='median',axis=0)
    X[:] = imputer.fit_transform(X)
    #print("The X matrix now has shape (%dx%d) (molecules by non-NaN good molecular descriptors)" % X.shape)
    return X,imputer
//  [python]
def impute_X(X):
    imputer = SimpleImputer(missing_values=np.nan,strategy='median')
    X[:] = imputer.fit_transform(X)
    return X,imputer
//  [python]
def fill_na(self, X, col_list, stragegy = 'mean'):
		# 
		for feature in col_list:
			X[feature] = Imputer(stragegy=stragegy).fit_transform(X[feature])
		return X
//  [python]
def fill_na(self, X, col_list, stragegy = 'mean'):
		# 
		for feature in col_list:
			X[feature] = SimpleImputer(strategy=stragegy).fit_transform(X[[feature]])
		return X
//  [python]
def drop_impute_and_scale_dataframe(df, scaling="std", imputing="mean", dropna="all"):
    """Impute missing values with mean and scale data included in pandas dataframe.

    Parameters
    ----------
    df : pandas dataframe
        dataframe to process
    scaling : string
        String describing type of scaling to apply.
        'maxabs' [-1,1], 'minmax' [0,1], 'std', or None, optional
        (Default 'std')
    imputing : string
        String describing type of imputation to apply.
        'mean' replace missing values with mean value along the column,
        'median' replace missing values with median value along the column,
        'most_frequent' replace missing values with most frequent value along column
        (Default: 'mean').
    dropna : string
        String describing strategy for handling missing values.
        'all' if all values are NA, drop that column.
        'any' if any NA values are present, dropt that column.
        (Default: 'all').

    Return
    ----------
    Returns the data frame after handling missing values and scaling.

    """

    if dropna:
        df = df.dropna(axis=1, how=dropna)
    else:
        empty_cols = df.columns[df.notnull().sum() == 0]
        df[empty_cols] = 0

    if imputing is None or imputing.lower() == "none":
        mat = df.values
    else:
        # imputer = Imputer(strategy=imputing, axis=0)
        # imputer = SimpleImputer(strategy=imputing)
        # Next line is from conditional import. axis=0 is default
        # in old version so it is not necessary.
        imputer = Imputer(strategy=imputing)
        mat = imputer.fit_transform(df.values)

    if scaling is None or scaling.lower() == "none":
        return pd.DataFrame(mat, columns=df.columns)

    if scaling == "maxabs":
        scaler = MaxAbsScaler()
    elif scaling == "minmax":
        scaler = MinMaxScaler()
    else:
        scaler = StandardScaler()

    mat = scaler.fit_transform(mat)
    df = pd.DataFrame(mat, columns=df.columns)

    return df
//  [python]
def drop_impute_and_scale_dataframe(df, scaling="std", imputing="mean", dropna="all"):
    """Impute missing values with mean and scale data included in pandas dataframe.

    Parameters
    ----------
    df : pandas dataframe
        dataframe to process
    scaling : string
        String describing type of scaling to apply.
        'maxabs' [-1,1], 'minmax' [0,1], 'std', or None, optional
        (Default 'std')
    imputing : string
        String describing type of imputation to apply.
        'mean' replace missing values with mean value along the column,
        'median' replace missing values with median value along the column,
        'most_frequent' replace missing values with most frequent value along column
        (Default: 'mean').
    dropna : string
        String describing strategy for handling missing values.
        'all' if all values are NA, drop that column.
        'any' if any NA values are present, dropt that column.
        (Default: 'all').

    Return
    ----------
    Returns the data frame after handling missing values and scaling.

    """

    if dropna:
        df = df.dropna(axis=1, how=dropna)
    else:
        empty_cols = df.columns[df.notnull().sum() == 0]
        df[empty_cols] = 0

    if imputing is None or imputing.lower() == "none":
        mat = df.values
    else:
        imputer = SimpleImputer(strategy=imputing)
        mat = imputer.fit_transform(df.values)

    if scaling is None or scaling.lower() == "none":
        return pd.DataFrame(mat, columns=df.columns)

    if scaling == "maxabs":
        scaler = MaxAbsScaler()
    elif scaling == "minmax":
        scaler = MinMaxScaler()
    else:
        scaler = StandardScaler()

    mat = scaler.fit_transform(mat)
    df = pd.DataFrame(mat, columns=df.columns)

    return df
//  [python]
def impute_and_scale(df, scaling="std", keepcols=None):
    """Impute missing values with mean and scale data included in pandas dataframe.

    Parameters
    ----------
    df : pandas dataframe
        dataframe to impute and scale
    scaling : 'maxabs' [-1,1], 'minmax' [0,1], 'std', or None, optional (default 'std')
        type of scaling to apply
    """

    if keepcols is None:
        df = df.dropna(axis=1, how="all")
    else:
        df = df[keepcols].copy()
        all_na_cols = df.columns[df.isna().all()]
        df[all_na_cols] = 0

    imputer = Imputer(strategy="mean")
    mat = imputer.fit_transform(df)

    if scaling is None or scaling.lower() == "none":
        return pd.DataFrame(mat, columns=df.columns)

    if scaling == "maxabs":
        scaler = MaxAbsScaler()
    elif scaling == "minmax":
        scaler = MinMaxScaler()
    else:
        scaler = StandardScaler()

    mat = scaler.fit_transform(mat)

    df = pd.DataFrame(mat, columns=df.columns)

    return df
//  [python]
def impute_and_scale(df, scaling="std", keepcols=None):
    """Impute missing values with mean and scale data included in pandas dataframe.

    Parameters
    ----------
    df : pandas dataframe
        dataframe to impute and scale
    scaling : 'maxabs' [-1,1], 'minmax' [0,1], 'std', or None, optional (default 'std')
        type of scaling to apply
    """

    if keepcols is None:
        df = df.dropna(axis=1, how="all")
    else:
        df = df[keepcols].copy()
        all_na_cols = df.columns[df.isna().all()]
        df[all_na_cols] = 0

    imputer = SimpleImputer(strategy="mean")
    mat = imputer.fit_transform(df)

    if scaling is None or scaling.lower() == "none":
        return pd.DataFrame(mat, columns=df.columns)

    if scaling == "maxabs":
        scaler = MaxAbsScaler()
    elif scaling == "minmax":
        scaler = MinMaxScaler()
    else:
        scaler = StandardScaler()

    mat = scaler.fit_transform(mat)

    df = pd.DataFrame(mat, columns=df.columns)

    return df
//  [python]
def test_pipeline_column_transformer_string(self):
        """
        TODO: Hummingbird does not yet support strings in this context. Should raise error.
        When this feature is complete, change this test.
        """
        # fit
        titanic_url = "https://raw.githubusercontent.com/amueller/scipy-2017-sklearn/091d371/notebooks/datasets/titanic3.csv"
        data = pandas.read_csv(titanic_url)
        X = data.drop("survived", axis=1)
        y = data["survived"]
        # SimpleImputer on string is not available for string
        # in ONNX-ML specifications.
        # So we do it beforehand.
        X['pclass'] = X['pclass'].fillna('missing')

        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

        numeric_features = ["age", "fare"]
        numeric_transformer = Pipeline(steps=[("imputer", SimpleImputer(strategy="median")), ("scaler", StandardScaler())])

        categorical_features = ["pclass"]
        categorical_transformer = Pipeline(steps=[("onehot", OneHotEncoder(handle_unknown="ignore"))])

        preprocessor = ColumnTransformer(
            transformers=[
                ("num", numeric_transformer, numeric_features),
                ("cat", categorical_transformer, categorical_features),
            ]
        )

        clf = Pipeline(steps=[("preprocessor", preprocessor), ("classifier", LogisticRegression(solver="liblinear"))])

        to_drop = {"parch", "sibsp", "cabin", "ticket", "name", "body", "home.dest", "boat", "sex", "embarked"}

        X_train = X_train.copy()
        X_test = X_test.copy()
        X_train["pclass"] = X_train["pclass"].astype(np.int64)
        X_test["pclass"] = X_test["pclass"].astype(np.int64)
        X_train = X_train.drop(to_drop, axis=1)
        X_test = X_test.drop(to_drop, axis=1)

        clf.fit(X_train, y_train)

        torch_model = hummingbird.ml.convert(clf, "torch", X_test)

        self.assertTrue(torch_model is not None)

        np.testing.assert_allclose(
            clf.predict(X_test), torch_model.predict(X_test), rtol=1e-06, atol=1e-06,
        )
//  [python]
def test_imputer(self):
        try:
            model = Imputer(missing_values="NaN", strategy="mean", axis=0)
        except TypeError:
            model = Imputer(missing_values=np.nan, strategy="mean")
            model.axis = 0
        data = [[1, 2], [np.nan, 3], [7, 6]]
        model.fit(data)
        from onnxmltools.convert.coreml.convert import convert
        import coremltools  # noqa

        try:
            model_coreml = coremltools.converters.sklearn.convert(model)
        except ValueError as e:
            if "not supported" in str(e):
                # Python 2.7 + scikit-learn 0.22
                return
        model_onnx = convert(model_coreml.get_spec(), target_opset=TARGET_OPSET)
        self.assertTrue(model_onnx is not None)
        dump_data_and_model(
            np.array(data, dtype=np.float32),
            model,
            model_onnx,
            basename="CmlImputerMeanFloat32",
        )
//  [python]
def test_imputer(self):
        try:
            model = SimpleImputer(missing_values=np.nan, strategy="mean")
            model.fit([[1, 2], [np.nan, 3], [7, 6]])
        except ValueError:
            model = SimpleImputer(missing_values="NaN", strategy="mean")
            model.fit([[1, 2], [np.nan, 3], [7, 6]])

        data = [[1, 2], [np.nan, 3], [7, 6]]
        model.fit(data)
        from onnxmltools.convert.coreml.convert import convert
        import coremltools  # noqa

        try:
            model_coreml = coremltools.converters.sklearn.convert(model)
        except ValueError as e:
            if "not supported" in str(e):
                # Python 2.7 + scikit-learn 0.22
                return
        model_onnx = convert(model_coreml.get_spec(), target_opset=TARGET_OPSET)
        self.assertTrue(model_onnx is not None)
        dump_data_and_model(
            np.array(data, dtype=np.float32),
            model,
            model_onnx,
            basename="CmlImputerMeanFloat32",
        )
//  [python]
def l1_regr(genotypes, covariate_df, formula, C=0.1):
    from sklearn.linear_model import LogisticRegression
    from sklearn.preprocessing import Imputer, StandardScaler
    assert genotypes.shape[0] == covariate_df.shape[0]

    genotypes.index = covariate_df.index
    genotypes.ix[:, :] = Imputer(missing_values="NaN",
                                 strategy="most_frequent",
                                 axis=0).fit_transform(genotypes)
    try:
        y, covariates = patsy.dmatrices(str(formula), covariate_df,
                return_type='dataframe')
    except:
        return

    covariates.drop([x for x in covariates.columns if 'genotype' in x], inplace=True, axis=1)
    covariates.ix[:, :] = StandardScaler().fit_transform(covariates)
    X = pd.concat((covariates, genotypes), axis=1)
    #X.to_csv('/tmp/X.csv')
    #y.to_csv('/tmp/y.csv')

    assert X.shape[0] == covariate_df.shape[0]
    # or drop variants with missing data.
    #X = X.dropna(axis=1)
    tol, do_break = 1e-4, True
    for p in ('l1', 'l2'):
        for c in (1e-3, 0.01, 0.1, 0.25, 0.5, 1, 2, 10, 1000, 1e8):
            clf = LogisticRegression(C=c, penalty=p, tol=tol,
                        fit_intercept=False)
            try:
                clf.fit_transform(X, y.squeeze())
            except ValueError:
                continue
            coefs = clf.coef_.ravel()
            variants = list(X.columns[coefs != 0])

            scores = list(coefs[coefs != 0])
            #n = X_new.shape[1]
            n = len(variants)
            print("penalty: {p} C: {c} n_covariates: {n}".format(**locals()), file=sys.stderr)
            print([(v, "%.3f" % s) for s, v in sorted(zip(np.abs(scores), variants),
                reverse=True)], file=sys.stderr)
            break
        else:
            do_break = False
        if do_break: break
//  [python]
def l1_regr(genotypes, covariate_df, formula, C=0.1):
    from sklearn.linear_model import LogisticRegression
    from sklearn.impute import SimpleImputer
    from sklearn.preprocessing import StandardScaler
    assert genotypes.shape[0] == covariate_df.shape[0]

    genotypes.index = covariate_df.index
    genotypes.ix[:, :] = SimpleImputer(missing_values="NaN",
                                 strategy="most_frequent",
                                 axis=0).fit_transform(genotypes)
    try:
        y, covariates = patsy.dmatrices(str(formula), covariate_df,
                return_type='dataframe')
    except:
        return

    covariates.drop([x for x in covariates.columns if 'genotype' in x], inplace=True, axis=1)
    covariates.ix[:, :] = StandardScaler().fit_transform(covariates)
    X = pd.concat((covariates, genotypes), axis=1)
    #X.to_csv('/tmp/X.csv')
    #y.to_csv('/tmp/y.csv')

    assert X.shape[0] == covariate_df.shape[0]
    # or drop variants with missing data.
    #X = X.dropna(axis=1)
    tol, do_break = 1e-4, True
    for p in ('l1', 'l2'):
        for c in (1e-3, 0.01, 0.1, 0.25, 0.5, 1, 2, 10, 1000, 1e8):
            clf = LogisticRegression(C=c, penalty=p, tol=tol,
                        fit_intercept=False)
            try:
                clf.fit_transform(X, y.squeeze())
            except ValueError:
                continue
            coefs = clf.coef_.ravel()
            variants = list(X.columns[coefs != 0])

            scores = list(coefs[coefs != 0])
            #n = X_new.shape[1]
            n = len(variants)
            print("penalty: {p} C: {c} n_covariates: {n}".format(**locals()), file=sys.stderr)
            print([(v, "%.3f" % s) for s, v in sorted(zip(np.abs(scores), variants),
                reverse=True)], file=sys.stderr)
            break
        else:
            do_break = False
        if do_break: break
//  [python]
def test_imputer_vs_sklearn():
    # Compare msmbuilder.preprocessing.Imputer
    # with sklearn.preprocessing.Imputer

    imputerr = ImputerR()
    imputerr.fit(np.concatenate(trajs))

    imputer = Imputer()
    imputer.fit(trajs)

    y_ref1 = imputerr.transform(trajs[0])
    y1 = imputer.transform(trajs)[0]

    np.testing.assert_array_almost_equal(y_ref1, y1)
//  [python]
def test_imputer_vs_sklearn():
    # Compare msmbuilder.preprocessing.Imputer
    # with sklearn.preprocessing.Imputer

    imputerr = ImputerR()
    imputerr.fit(np.concatenate(trajs))

    imputer = SimpleImputer()
    imputer.fit(trajs)

    y_ref1 = imputerr.transform(trajs[0])
    y1 = imputer.transform(trajs)[0]

    np.testing.assert_array_almost_equal(y_ref1, y1)
//  [python]
def median_imputer():
    return Imputer(strategy='median')
//  [python]
def median_imputer():
    return SimpleImputer(strategy='median')
//  [python]
def test_pipeline_column_transformer(self):
        iris = datasets.load_iris()
        X = iris.data[:, :3]
        y = iris.target
        X_train = pandas.DataFrame(X, columns=["vA", "vB", "vC"])
        X_train["vcat"] = X_train["vA"].apply(lambda x: "cat1" if x > 0.5 else "cat2")
        X_train["vcat2"] = X_train["vB"].apply(lambda x: "cat3" if x > 0.5 else "cat4")
        y_train = y % 2
        numeric_features = [0, 1, 2]  # ["vA", "vB", "vC"]
        categorical_features = [3, 4]  # ["vcat", "vcat2"]

        classifier = LogisticRegression(
            C=0.01,
            class_weight=dict(zip([False, True], [0.2, 0.8])),
            n_jobs=1,
            max_iter=10,
            solver="lbfgs",
            tol=1e-3,
        )

        numeric_transformer = Pipeline(
            steps=[
                ("imputer", SimpleImputer(strategy="median")),
                ("scaler", StandardScaler()),
            ]
        )

        categorical_transformer = Pipeline(
            steps=[
                (
                    "onehot",
                    OneHotEncoder(sparse_output=True, handle_unknown="ignore"),
                ),
                (
                    "tsvd",
                    TruncatedSVD(n_components=1, algorithm="arpack", tol=1e-4),
                ),
            ]
        )

        preprocessor = ColumnTransformer(
            transformers=[
                ("num", numeric_transformer, numeric_features),
                ("cat", categorical_transformer, categorical_features),
            ]
        )

        model = Pipeline(
            steps=[("precprocessor", preprocessor), ("classifier", classifier)]
        )

        model.fit(X_train, y_train)
        initial_type = [
            ("numfeat", FloatTensorType([None, 3])),
            ("strfeat", StringTensorType([None, 2])),
        ]

        X_train = X_train[:11]
        model_onnx = convert_sklearn(
            model, initial_types=initial_type, target_opset=TARGET_OPSET
        )

        dump_data_and_model(
            X_train,
            model,
            model_onnx,
            basename="SklearnPipelineColumnTransformerPipeliner",
        )

        if __name__ == "__main__":
            try:
                from onnx.tools.net_drawer import GetPydotGraph, GetOpNodeProducer
            except ImportError:
                return

            pydot_graph = GetPydotGraph(
                model_onnx.graph,
                name=model_onnx.graph.name,
                rankdir="TP",
                node_producer=GetOpNodeProducer("docstring"),
            )
            pydot_graph.write_dot("graph.dot")

            import os

            os.system("dot -O -G=300 -Tpng graph.dot")
//  [python]
def process(site):

	# Testing data
	test_df = test[test['ForecastId'] == site].sort_values(['Timestamp', 'Distance'])
	test_df = test_df.drop_duplicates(['Timestamp'], keep='first')

	# Training data
	train_df = train[train['ForecastId'] == site].sort_values(['Timestamp', 'Distance'])
	train_df = train_df.drop_duplicates(['Timestamp'], keep='first')

	# Only use past training data
	train_df = train_df[train_df['Timestamp'] < test_df['Timestamp'].min()]

	# If all training temperatures are missing, drop temperatures from both training and testing
	if (np.all(np.isnan(train_df['Temperature']))) or (np.all(np.isnan(test_df['Temperature']))):
		train_df = train_df.drop(labels = 'Temperature', axis=1)
		test_df = test_df.drop(labels= 'Temperature', axis=1)

	# Otherwise impute the missing temperatures
	else:
		temp_median_imputer = Imputer(missing_values='NaN', strategy='median', axis = 0)
		temp_median_imputer.fit(train_df[['Temperature']])
		train_df['Temperature'] = temp_median_imputer.transform(train_df[['Temperature']])
		test_df['Temperature'] = temp_median_imputer.transform(test_df[['Temperature']])

	# Impute the missing values
	value_median_imputer = Imputer(missing_values='NaN', strategy='median', axis=0)
	value_median_imputer.fit(train_df[['Value']])

	if pd.isnull(train_df['Value']).all():
		train_df['Value'] = 0
	else:
		train_df['Value'] = value_median_imputer.transform(train_df[['Value']])

	# Find the minimum date for converting timestamp to numeric
	min_date = min(train_df['Timestamp'])

	# Convert timestamp to numeric
	train_df['Timestamp'] = (train_df['Timestamp'] - min_date).dt.total_seconds()
	test_df['Timestamp']  = (test_df['Timestamp'] - min_date).dt.total_seconds()

	# Interval between measurements
	train_df['time_diff'] = train_df['Timestamp'].diff().fillna(0)
	test_df['time_diff'] = test_df['Timestamp'].diff().fillna(0)

	# Extract labels
	train_labels = train_df['Value']

	# Drop columns
	train_df = train_df.drop(columns = ['Distance', 'SiteId', 'ForecastId', 'Value'])
	test_df =   test_df.drop(columns = ['Distance', 'SiteId', 'ForecastId', 'Value'])


	# Scale the features between 0 and 1 (best practice for ML)
	scaler = MinMaxScaler()

	train_df.ix[:, :] = scaler.fit_transform(train_df.ix[:, :])
	test_df.ix[:, :] = scaler.transform(test_df.ix[:, :])

	return train_df, train_labels, test_df
//  [python]
def process(site):

	# Testing data
	test_df = test[test['ForecastId'] == site].sort_values(['Timestamp', 'Distance'])
	test_df = test_df.drop_duplicates(['Timestamp'], keep='first')

	# Training data
	train_df = train[train['ForecastId'] == site].sort_values(['Timestamp', 'Distance'])
	train_df = train_df.drop_duplicates(['Timestamp'], keep='first')

	# Only use past training data
	train_df = train_df[train_df['Timestamp'] < test_df['Timestamp'].min()]

	# If all training temperatures are missing, drop temperatures from both training and testing
	if (np.all(np.isnan(train_df['Temperature']))) or (np.all(np.isnan(test_df['Temperature']))):
		train_df = train_df.drop(labels = 'Temperature', axis=1)
		test_df = test_df.drop(labels= 'Temperature', axis=1)

	# Otherwise impute the missing temperatures
	else:
		temp_median_imputer = SimpleImputer(missing_values=np.nan, strategy='median')
		train_df['Temperature'] = temp_median_imputer.fit_transform(train_df[['Temperature']])
		test_df['Temperature'] = temp_median_imputer.transform(test_df[['Temperature']])

	# Impute the missing values
	value_median_imputer = SimpleImputer(missing_values=np.nan, strategy='median')
	value_median_imputer.fit(train_df[['Value']])

	if pd.isnull(train_df['Value']).all():
		train_df['Value'] = 0
	else:
		train_df['Value'] = value_median_imputer.transform(train_df[['Value']])

	# Find the minimum date for converting timestamp to numeric
	min_date = min(train_df['Timestamp'])

	# Convert timestamp to numeric
	train_df['Timestamp'] = (train_df['Timestamp'] - min_date).dt.total_seconds()
	test_df['Timestamp']  = (test_df['Timestamp'] - min_date).dt.total_seconds()

	# Interval between measurements
	train_df['time_diff'] = train_df['Timestamp'].diff().fillna(0)
	test_df['time_diff'] = test_df['Timestamp'].diff().fillna(0)

	# Extract labels
	train_labels = train_df['Value']

	# Drop columns
	train_df = train_df.drop(columns = ['Distance', 'SiteId', 'ForecastId', 'Value'])
	test_df =   test_df.drop(columns = ['Distance', 'SiteId', 'ForecastId', 'Value'])


	# Scale the features between 0 and 1 (best practice for ML)
	scaler = MinMaxScaler()

	train_df.ix[:, :] = scaler.fit_transform(train_df.ix[:, :])
	test_df.ix[:, :] = scaler.transform(test_df.ix[:, :])

	return train_df, train_labels, test_df
//  [python]
def mean_imputer():
    return Imputer(strategy='mean')
//  [python]
def mean_imputer():
    return SimpleImputer(strategy='mean')
//  [python]
def __init__(self):
        self.enc = {}
        self.cat2idx = {}
        self.dist = {}
        self.scaler = StandardScaler()
        self.model = None
        self.best_alpha = 1
        self.good_cols = None
        self.good_idxs = None
        self.train_shape = None
        self.do_sample = False
        self.imputer = Imputer()
        self.size =1000
//  [python]
def __init__(self):
        self.enc = {}
        self.cat2idx = {}
        self.dist = {}
        self.scaler = StandardScaler()
        self.model = None
        self.best_alpha = 1
        self.good_cols = None
        self.good_idxs = None
        self.train_shape = None
        self.do_sample = False
        self.imputer = SimpleImputer()
        self.size =1000
//  [python]
def load_datasets(feature_paths, label_paths):
    feature = np.ndarray(shape=(0, 41))
    label = np.ndarray(shape=(0, 1))
    for file in feature_paths:
        df = pd.read_table(file, delimiter=',', na_values='?', header=None)
        imp = Imputer(missing_values='NaN', strategy='mean', axis=0)
        imp.fit(df)
        df = imp.transform(df)
        feature = np.concatenate((feature, df))

    for file in label_paths:
        df = pd.read_table(file, header=None)
        label = np.concatenate((label, df))

    label = np.ravel(label)
    return feature, label
//  [python]
def load_datasets(feature_paths, label_paths):
    feature = np.ndarray(shape=(0, 41))
    label = np.ndarray(shape=(0, 1))
    for file in feature_paths:
        df = pd.read_table(file, delimiter=',', na_values='?', header=None)
        imp = SimpleImputer(missing_values=np.nan, strategy='mean')
        df = imp.fit_transform(df)
        feature = np.concatenate((feature, df))

    for file in label_paths:
        df = pd.read_table(file, header=None)
        label = np.concatenate((label, df))

    label = np.ravel(label)
    return feature, label
//  [python]
def test_relieff_pipeline_multiclass():
    """Check: Data (Multiclass Endpoint): ReliefF works in a sklearn pipeline """
    np.random.seed(49082)

    clf = make_pipeline(ReliefF(n_features_to_select=2, n_neighbors=10),
                        Imputer(),
                        RandomForestClassifier(n_estimators=100, n_jobs=-1))

    assert np.mean(cross_val_score(clf, features_multiclass,
                                   labels_multiclass, cv=3, n_jobs=-1)) > 0.7
//  [python]
def test_relieff_pipeline_multiclass():
    """Check: Data (Multiclass Endpoint): ReliefF works in a sklearn pipeline """
    np.random.seed(49082)

    clf = make_pipeline(ReliefF(n_features_to_select=2, n_neighbors=10),
                        SimpleImputer(),
                        RandomForestClassifier(n_estimators=100, n_jobs=-1))

    assert np.mean(cross_val_score(clf, features_multiclass,
                                   labels_multiclass, cv=3, n_jobs=-1)) > 0.7
//  [python]
def test_surf_pipeline_multiclass():
    """Check: Data (Multiclass Endpoint): SURF works in a sklearn pipeline"""
    np.random.seed(240932)

    clf = make_pipeline(SURF(n_features_to_select=2),
                        Imputer(),
                        RandomForestClassifier(n_estimators=100, n_jobs=-1))

    assert np.mean(cross_val_score(clf, features_multiclass,
                                   labels_multiclass, cv=3, n_jobs=-1)) > 0.7
//  [python]
def test_surf_pipeline_multiclass():
    """Check: Data (Multiclass Endpoint): SURF works in a sklearn pipeline"""
    np.random.seed(240932)

    clf = make_pipeline(SURF(n_features_to_select=2),
                        SimpleImputer(),
                        RandomForestClassifier(n_estimators=100, n_jobs=-1))

    assert np.mean(cross_val_score(clf, features_multiclass,
                                   labels_multiclass, cv=3, n_jobs=-1)) > 0.7
//  [python]
def test_surfstar_pipeline_multiclass():
    """Check: Data (Multiclass Endpoint): SURF* works in a sklearn pipeline"""
    np.random.seed(9238745)

    clf = make_pipeline(SURFstar(n_features_to_select=2),
                        Imputer(),
                        RandomForestClassifier(n_estimators=100, n_jobs=-1))

    assert np.mean(cross_val_score(clf, features_multiclass,
                                   labels_multiclass, cv=3, n_jobs=-1)) > 0.7
//  [python]
def test_surfstar_pipeline_multiclass():
    """Check: Data (Multiclass Endpoint): SURF* works in a sklearn pipeline"""
    np.random.seed(9238745)

    clf = make_pipeline(SURFstar(n_features_to_select=2),
                        SimpleImputer(),
                        RandomForestClassifier(n_estimators=100, n_jobs=-1))

    assert np.mean(cross_val_score(clf, features_multiclass,
                                   labels_multiclass, cv=3, n_jobs=-1)) > 0.7
//  [python]
def __init__(self):
        self.data = None
        self.X_train = None
        self.Y_train = None
        self.X_test = None
        self.Y_test = None
        self.clf = None
        category_binarizer = OnceFittedLabelBinarizer()
        country_binarizer = OnceFittedLabelBinarizer()
        state_binarizer = OnceFittedLabelBinarizer()
        self.category_mapper = DataFrameMapper([
            (['category_code'], [CategoricalImputer(), category_binarizer]),
            (['country_code'], [CategoricalImputer(), country_binarizer]),
            (['state_code'], [CategoricalImputer(), state_binarizer]),
        ])
        self.mapper = DataFrameMapper([
            (['category_code'], [CategoricalImputer(), category_binarizer], {'alias': 'category'}),
            (['country_code'], [CategoricalImputer(), country_binarizer], {'alias': 'country'}),
            (['state_code'], [CategoricalImputer(), state_binarizer], {'alias': 'state'}),
            (['mba_degree'], [ValueImputer(0), StandardScaler()]),
            (['phd_degree'], [ValueImputer(0), StandardScaler()]),
            (['ms_degree'], [ValueImputer(0), StandardScaler()]),
            (['other_degree'], [ValueImputer(0)]),
            (['age'], [Imputer(), StandardScaler()]),
            (['offices'], [ValueImputer(1.0), StandardScaler()]),
            (['products_number'], [ValueImputer(1.0), StandardScaler()]),
            (['average_funded', 'average_participants'], [ParticipantsImputer(), StandardScaler()],
             {'alias': 'average_participants'}),
            (['total_rounds'], None),
            (['ipo'], None),
            (['is_closed'], None),
            (['total_rounds', 'average_funded'], [FundImputer(), StandardScaler()], {'alias': 'average_funded'}),
            (['acquired_companies'], [ValueImputer(0)]),
        ])
        SVC_C_grid = [10 ** i for i in range(-3, 4)]
        SVC_gamma_grid = [10 ** i for i in range(-3, 1)] + ['auto']
        MLP_hidden_layer_sizes = [[25], [50], [75], [100], [50, 25], [75, 50], [100, 75], [75, 50, 25], [100, 75, 50]]
        MLP_activation = ['logistic', 'tanh', 'relu']
        self.grid = [{'clf': [GradientBoostingClassifier()], 'clf__n_estimators': [20 * i for i in range(5, 8)],
                 'clf__max_depth': [i + 3 for i in range(2, 6)]},
                {'clf': [SVC(kernel='rbf', class_weight='balanced')], 'clf__C': SVC_C_grid,
                 'clf__gamma': SVC_gamma_grid},
                {'clf': [SVC(kernel='poly', class_weight='balanced')], 'clf__C': SVC_C_grid,
                 'clf__gamma': SVC_gamma_grid,
                 'clf__degree': list(range(3, 6))},
                {'clf': [MLPClassifier()], 'clf__hidden_layer_sizes': MLP_hidden_layer_sizes,
                 'clf__activation': MLP_activation,
                 'clf__alpha': [10 ** i for i in range(-1, 3)]}]
//  [python]
def __init__(self):
        self.data = None
        self.X_train = None
        self.Y_train = None
        self.X_test = None
        self.Y_test = None
        self.clf = None
        category_binarizer = OnceFittedLabelBinarizer()
        country_binarizer = OnceFittedLabelBinarizer()
        state_binarizer = OnceFittedLabelBinarizer()
        self.category_mapper = DataFrameMapper([
            (['category_code'], [CategoricalImputer(), category_binarizer]),
            (['country_code'], [CategoricalImputer(), country_binarizer]),
            (['state_code'], [CategoricalImputer(), state_binarizer]),
        ])
        self.mapper = DataFrameMapper([
            (['category_code'], [CategoricalImputer(), category_binarizer], {'alias': 'category'}),
            (['country_code'], [CategoricalImputer(), country_binarizer], {'alias': 'country'}),
            (['state_code'], [CategoricalImputer(), state_binarizer], {'alias': 'state'}),
            (['mba_degree'], [ValueImputer(0), StandardScaler()]),
            (['phd_degree'], [ValueImputer(0), StandardScaler()]),
            (['ms_degree'], [ValueImputer(0), StandardScaler()]),
            (['other_degree'], [ValueImputer(0)]),
            (['age'], [SimpleImputer(), StandardScaler()]),
            (['offices'], [ValueImputer(1.0), StandardScaler()]),
            (['products_number'], [ValueImputer(1.0), StandardScaler()]),
            (['average_funded', 'average_participants'], [ParticipantsImputer(), StandardScaler()],
             {'alias': 'average_participants'}),
            (['total_rounds'], None),
            (['ipo'], None),
            (['is_closed'], None),
            (['total_rounds', 'average_funded'], [FundImputer(), StandardScaler()], {'alias': 'average_funded'}),
            (['acquired_companies'], [ValueImputer(0)]),
        ])
        SVC_C_grid = [10 ** i for i in range(-3, 4)]
        SVC_gamma_grid = [10 ** i for i in range(-3, 1)] + ['auto']
        MLP_hidden_layer_sizes = [[25], [50], [75], [100], [50, 25], [75, 50], [100, 75], [75, 50, 25], [100, 75, 50]]
        MLP_activation = ['logistic', 'tanh', 'relu']
        self.grid = [{'clf': [GradientBoostingClassifier()], 'clf__n_estimators': [20 * i for i in range(5, 8)],
                 'clf__max_depth': [i + 3 for i in range(2, 6)]},
                {'clf': [SVC(kernel='rbf', class_weight='balanced')], 'clf__C': SVC_C_grid,
                 'clf__gamma': SVC_gamma_grid},
                {'clf': [SVC(kernel='poly', class_weight='balanced')], 'clf__C': SVC_C_grid,
                 'clf__gamma': SVC_gamma_grid,
                 'clf__degree': list(range(3, 6))},
                {'clf': [MLPClassifier()], 'clf__hidden_layer_sizes': MLP_hidden_layer_sizes,
                 'clf__activation': MLP_activation,
                 'clf__alpha': [10 ** i for i in range(-1, 3)]}]
//  [python]
def test_multisurf_pipeline_multiclass():
    """Check: Data (Multiclass Endpoint): MultiSURF works in a sklearn pipeline"""
    np.random.seed(320931)

    clf = make_pipeline(MultiSURF(n_features_to_select=2),
                        Imputer(),
                        RandomForestClassifier(n_estimators=100, n_jobs=-1))

    assert np.mean(cross_val_score(clf, features_multiclass,
                                   labels_multiclass, cv=3, n_jobs=-1)) > 0.7
//  [python]
def test_multisurf_pipeline_multiclass():
    """Check: Data (Multiclass Endpoint): MultiSURF works in a sklearn pipeline"""
    np.random.seed(320931)

    clf = make_pipeline(MultiSURF(n_features_to_select=2),
                        SimpleImputer(),
                        RandomForestClassifier(n_estimators=100, n_jobs=-1))

    assert np.mean(cross_val_score(clf, features_multiclass,
                                   labels_multiclass, cv=3, n_jobs=-1)) > 0.7
//  [python]
def test_multisurfstar_pipeline_multiclass():
    """Check: Data (Multiclass Endpoint): MultiSURF* works in a sklearn pipeline"""
    np.random.seed(320931)

    clf = make_pipeline(MultiSURFstar(n_features_to_select=2),
                        Imputer(),
                        RandomForestClassifier(n_estimators=100, n_jobs=-1))

    assert np.mean(cross_val_score(clf, features_multiclass,
                                   labels_multiclass, cv=3, n_jobs=-1)) > 0.7
//  [python]
def test_multisurfstar_pipeline_multiclass():
    """Check: Data (Multiclass Endpoint): MultiSURF* works in a sklearn pipeline"""
    np.random.seed(320931)

    clf = make_pipeline(MultiSURFstar(n_features_to_select=2),
                        SimpleImputer(),
                        RandomForestClassifier(n_estimators=100, n_jobs=-1))

    assert np.mean(cross_val_score(clf, features_multiclass,
                                   labels_multiclass, cv=3, n_jobs=-1)) > 0.7
//  [python]
def test_relieff_pipeline_missing_values():
    """Check: Data (Missing Values): ReliefF works in a sklearn pipeline"""
    np.random.seed(49082)

    clf = make_pipeline(ReliefF(n_features_to_select=2, n_neighbors=10),
                        Imputer(),
                        RandomForestClassifier(n_estimators=100, n_jobs=-1))

    assert np.mean(cross_val_score(clf, features_missing_values,
                                   labels_missing_values, cv=3, n_jobs=-1)) > 0.7
//  [python]
def test_relieff_pipeline_missing_values():
    """Check: Data (Missing Values): ReliefF works in a sklearn pipeline"""
    np.random.seed(49082)

    clf = make_pipeline(ReliefF(n_features_to_select=2, n_neighbors=10),
                        SimpleImputer(),
                        RandomForestClassifier(n_estimators=100, n_jobs=-1))

    assert np.mean(cross_val_score(clf, features_missing_values,
                                   labels_missing_values, cv=3, n_jobs=-1)) > 0.7
//  [python]
def test_surf_pipeline_missing_values():
    """Check: Data (Missing Values): SURF works in a sklearn pipeline"""
    np.random.seed(240932)

    clf = make_pipeline(SURF(n_features_to_select=2),
                        Imputer(),
                        RandomForestClassifier(n_estimators=100, n_jobs=-1))

    assert np.mean(cross_val_score(clf, features_missing_values,
                                   labels_missing_values, cv=3, n_jobs=-1)) > 0.7
//  [python]
def test_surf_pipeline_missing_values():
    """Check: Data (Missing Values): SURF works in a sklearn pipeline"""
    np.random.seed(240932)

    clf = make_pipeline(SURF(n_features_to_select=2),
                        SimpleImputer(),
                        RandomForestClassifier(n_estimators=100, n_jobs=-1))

    assert np.mean(cross_val_score(clf, features_missing_values,
                                   labels_missing_values, cv=3, n_jobs=-1)) > 0.7
//  [python]
def test_surfstar_pipeline_missing_values():
    """Check: Data (Missing Values): SURF* works in a sklearn pipeline"""
    np.random.seed(9238745)

    clf = make_pipeline(SURFstar(n_features_to_select=2),
                        Imputer(),
                        RandomForestClassifier(n_estimators=100, n_jobs=-1))

    assert np.mean(cross_val_score(clf, features_missing_values,
                                   labels_missing_values, cv=3, n_jobs=-1)) > 0.7
//  [python]
def test_surfstar_pipeline_missing_values():
    """Check: Data (Missing Values): SURF* works in a sklearn pipeline"""
    np.random.seed(9238745)

    clf = make_pipeline(SURFstar(n_features_to_select=2),
                        SimpleImputer(),
                        RandomForestClassifier(n_estimators=100, n_jobs=-1))

    assert np.mean(cross_val_score(clf, features_missing_values,
                                   labels_missing_values, cv=3, n_jobs=-1)) > 0.7