jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def test_imputation_constant_object(marker):
    # Test imputation using the constant strategy on objects
    X = np.array([
        [marker, "a", "b", marker],
        ["c", marker, "d", marker],
        ["e", "f", marker, marker],
        ["g", "h", "i", marker]
    ], dtype=object)

    X_true = np.array([
        ["missing", "a", "b", "missing"],
        ["c", "missing", "d", "missing"],
        ["e", "f", "missing", "missing"],
        ["g", "h", "i", "missing"]
    ], dtype=object)

    imputer = Imputer(missing_values=marker, strategy="constant",
                            fill_value="missing")
    X_trans = imputer.fit_transform(X)

    assert_array_equal(X_trans, X_true)
//  [python]
def test_imputation_constant_object(marker):
    # Test imputation using the constant strategy on objects
    X = np.array([
        [marker, "a", "b", marker],
        ["c", marker, "d", marker],
        ["e", "f", marker, marker],
        ["g", "h", "i", marker]
    ], dtype=object)

    X_true = np.array([
        ["missing", "a", "b", "missing"],
        ["c", "missing", "d", "missing"],
        ["e", "f", "missing", "missing"],
        ["g", "h", "i", "missing"]
    ], dtype=object)

    imputer = SimpleImputer(missing_values=marker, strategy="constant",
                            fill_value="missing")
    X_trans = imputer.fit_transform(X)

    assert_array_equal(X_trans, X_true)
//  [python]
def test_imputation_constant_pandas(dtype):
    # Test imputation using the constant strategy on pandas df
    pd = pytest.importorskip("pandas")

    f = io.StringIO("Cat1,Cat2,Cat3,Cat4\n"
                    ",i,x,\n"
                    "a,,y,\n"
                    "a,j,,\n"
                    "b,j,x,")

    df = pd.read_csv(f, dtype=dtype)

    X_true = np.array([
        ["missing_value", "i", "x", "missing_value"],
        ["a", "missing_value", "y", "missing_value"],
        ["a", "j", "missing_value", "missing_value"],
        ["b", "j", "x", "missing_value"]
    ], dtype=object)

    imputer = Imputer(strategy="constant")
    X_trans = imputer.fit_transform(df)

    assert_array_equal(X_trans, X_true)
//  [python]
def test_imputation_constant_pandas(dtype):
    # Test imputation using the constant strategy on pandas df
    pd = pytest.importorskip("pandas")

    f = io.StringIO("Cat1,Cat2,Cat3,Cat4\n"
                    ",i,x,\n"
                    "a,,y,\n"
                    "a,j,,\n"
                    "b,j,x,")

    df = pd.read_csv(f, dtype=dtype)

    X_true = np.array([
        ["missing_value", "i", "x", "missing_value"],
        ["a", "missing_value", "y", "missing_value"],
        ["a", "j", "missing_value", "missing_value"],
        ["b", "j", "x", "missing_value"]
    ], dtype=object)

    imputer = SimpleImputer(strategy="constant")
    X_trans = imputer.fit_transform(df)

    assert_array_equal(X_trans, X_true)
//  [python]
def test_imputation_pipeline_grid_search():
    # Test imputation within a pipeline + gridsearch.
    X = sparse_random_matrix(100, 100, density=0.10)
    missing_values = X.data[0]

    pipeline = Pipeline([('imputer',
                          Imputer(missing_values=missing_values)),
                         ('tree',
                          tree.DecisionTreeRegressor(random_state=0))])

    parameters = {
        'imputer__strategy': ["mean", "median", "most_frequent"]
    }

    Y = sparse_random_matrix(100, 1, density=0.10).toarray()
    gs = GridSearchCV(pipeline, parameters)
    gs.fit(X, Y)
//  [python]
def test_imputation_pipeline_grid_search():
    # Test imputation within a pipeline + gridsearch.
    X = sparse_random_matrix(100, 100, density=0.10)
    missing_values = X.data[0]

    pipeline = Pipeline([('imputer',
                          SimpleImputer(missing_values=missing_values)),
                         ('tree',
                          tree.DecisionTreeRegressor(random_state=0))])

    parameters = {
        'imputer__strategy': ["mean", "median", "most_frequent"]
    }

    Y = sparse_random_matrix(100, 1, density=0.10).toarray()
    gs = GridSearchCV(pipeline, parameters)
    gs.fit(X, Y)
//  [python]
def test_simple_imputation_add_indicator_sparse_matrix(arr_type):
    X_sparse = arr_type([
        [np.nan, 1, 5],
        [2, np.nan, 1],
        [6, 3, np.nan],
        [1, 2, 9]
    ])
    X_true = np.array([
        [3., 1., 5., 1., 0., 0.],
        [2., 2., 1., 0., 1., 0.],
        [6., 3., 5., 0., 0., 1.],
        [1., 2., 9., 0., 0., 0.],
    ])

    imputer = Imputer(missing_values=np.nan, add_indicator=True)
    X_trans = imputer.fit_transform(X_sparse)

    assert sparse.issparse(X_trans)
    assert X_trans.shape == X_true.shape
    assert_allclose(X_trans.toarray(), X_true)
//  [python]
def test_simple_imputation_add_indicator_sparse_matrix(arr_type):
    X_sparse = arr_type([
        [np.nan, 1, 5],
        [2, np.nan, 1],
        [6, 3, np.nan],
        [1, 2, 9]
    ])
    X_true = np.array([
        [3., 1., 5., 1., 0., 0.],
        [2., 2., 1., 0., 1., 0.],
        [6., 3., 5., 0., 0., 1.],
        [1., 2., 9., 0., 0., 0.],
    ])

    imputer = SimpleImputer(missing_values=np.nan, add_indicator=True)
    X_trans = imputer.fit_transform(X_sparse)

    assert sparse.issparse(X_trans)
    assert X_trans.shape == X_true.shape
    assert_allclose(X_trans.toarray(), X_true)
//  [python]
def process_data(data):
        imputer = Imputer(missing_values=np.nan, strategy="mean")
        return pd.DataFrame(imputer.fit_transform(pd.DataFrame(data)))
//  [python]
def process_data(data):
        imputer = SimpleImputer(missing_values=np.nan, strategy="mean")
        return pd.DataFrame(imputer.fit_transform(pd.DataFrame(data)))
//  [python]
def test_iterative_imputer_missing_at_transform(strategy):
    rng = np.random.RandomState(0)
    n = 100
    d = 10
    X_train = rng.randint(low=0, high=3, size=(n, d))
    X_test = rng.randint(low=0, high=3, size=(n, d))

    X_train[:, 0] = 1  # definitely no missing values in 0th column
    X_test[0, 0] = 0  # definitely missing value in 0th column

    imputer = IterativeImputer(missing_values=0,
                               max_iter=1,
                               initial_strategy=strategy,
                               random_state=rng).fit(X_train)
    initial_imputer = Imputer(missing_values=0,
                                    strategy=strategy).fit(X_train)

    # if there were no missing values at time of fit, then imputer will
    # only use the initial imputer for that feature at transform
    assert_allclose(imputer.transform(X_test)[:, 0],
                    initial_imputer.transform(X_test)[:, 0])
//  [python]
def test_iterative_imputer_missing_at_transform(strategy):
    rng = np.random.RandomState(0)
    n = 100
    d = 10
    X_train = rng.randint(low=0, high=3, size=(n, d))
    X_test = rng.randint(low=0, high=3, size=(n, d))

    X_train[:, 0] = 1  # definitely no missing values in 0th column
    X_test[0, 0] = 0  # definitely missing value in 0th column

    imputer = IterativeImputer(missing_values=0,
                               max_iter=1,
                               initial_strategy=strategy,
                               random_state=rng).fit(X_train)
    initial_imputer = SimpleImputer(missing_values=0,
                                    strategy=strategy).fit(X_train)

    # if there were no missing values at time of fit, then imputer will
    # only use the initial imputer for that feature at transform
    assert_allclose(imputer.transform(X_test)[:, 0],
                    initial_imputer.transform(X_test)[:, 0])
//  [python]
def __init__(self, args_use, *args, **kwargs):
        super(MissedModel, self).__init__(*args, **kwargs)
        self.model = Imputer(
            missing_values=args_use["miss_value"],
            strategy=args_use["fill_method"],
            fill_value=args_use["fill_value"],
        )

        self.k = {}
        self.model_Name = "Missed"
//  [python]
def __init__(self, args_use, *args, **kwargs):
        super(MissedModel, self).__init__(*args, **kwargs)
        self.model = SimpleImputer(
            missing_values=args_use["miss_value"],
            strategy=args_use["fill_method"],
            fill_value=args_use["fill_value"],
        )

        self.k = {}
        self.model_Name = "Missed"
//  [python]
def test_imputation_copy():
    # Test imputation with copy
    X_orig = sparse_random_matrix(5, 5, density=0.75, random_state=0)

    # copy=True, dense => copy
    X = X_orig.copy().toarray()
    imputer = Imputer(missing_values=0, strategy="mean", copy=True)
    Xt = imputer.fit(X).transform(X)
    Xt[0, 0] = -1
    assert not np.all(X == Xt)

    # copy=True, sparse csr => copy
    X = X_orig.copy()
    imputer = Imputer(missing_values=X.data[0], strategy="mean",
                            copy=True)
    Xt = imputer.fit(X).transform(X)
    Xt.data[0] = -1
    assert not np.all(X.data == Xt.data)

    # copy=False, dense => no copy
    X = X_orig.copy().toarray()
    imputer = Imputer(missing_values=0, strategy="mean", copy=False)
    Xt = imputer.fit(X).transform(X)
    Xt[0, 0] = -1
    assert_array_almost_equal(X, Xt)

    # copy=False, sparse csc => no copy
    X = X_orig.copy().tocsc()
    imputer = Imputer(missing_values=X.data[0], strategy="mean",
                            copy=False)
    Xt = imputer.fit(X).transform(X)
    Xt.data[0] = -1
    assert_array_almost_equal(X.data, Xt.data)

    # copy=False, sparse csr => copy
    X = X_orig.copy()
    imputer = Imputer(missing_values=X.data[0], strategy="mean",
                            copy=False)
    Xt = imputer.fit(X).transform(X)
    Xt.data[0] = -1
    assert not np.all(X.data == Xt.data)
//  [python]
def test_imputation_copy():
    # Test imputation with copy
    X_orig = sparse_random_matrix(5, 5, density=0.75, random_state=0)

    # copy=True, dense => copy
    X = X_orig.copy().toarray()
    imputer = SimpleImputer(missing_values=0, strategy="mean", copy=True)
    Xt = imputer.fit(X).transform(X)
    Xt[0, 0] = -1
    assert not np.all(X == Xt)

    # copy=True, sparse csr => copy
    X = X_orig.copy()
    imputer = SimpleImputer(missing_values=X.data[0], strategy="mean",
                            copy=True)
    Xt = imputer.fit(X).transform(X)
    Xt.data[0] = -1
    assert not np.all(X.data == Xt.data)

    # copy=False, dense => no copy
    X = X_orig.copy().toarray()
    imputer = SimpleImputer(missing_values=0, strategy="mean", copy=False)
    Xt = imputer.fit(X).transform(X)
    Xt[0, 0] = -1
    assert_array_almost_equal(X, Xt)

    # copy=False, sparse csc => no copy
    X = X_orig.copy().tocsc()
    imputer = SimpleImputer(missing_values=X.data[0], strategy="mean",
                            copy=False)
    Xt = imputer.fit(X).transform(X)
    Xt.data[0] = -1
    assert_array_almost_equal(X.data, Xt.data)

    # copy=False, sparse csr => copy
    X = X_orig.copy()
    imputer = SimpleImputer(missing_values=X.data[0], strategy="mean",
                            copy=False)
    Xt = imputer.fit(X).transform(X)
    Xt.data[0] = -1
    assert not np.all(X.data == Xt.data)
//  [python]
cat_cols = ['TEAM_ABBREVIATION_HOME','SEASON']

# create training and testing DataFrames
X_train, X_test, y_train, y_test =  \
  train_test_split(nbagames[num_cols + cat_cols],\
  nbagames[['WL_HOME']], test_size=0.2, random_state=0)

# setup column transformations
ohe = OneHotEncoder(drop='first', sparse=False)

cattrans = make_pipeline(ohe)
standtrans = make_pipeline(OutlierTrans(2),
  Imputer(strategy="median"), StandardScaler())
//  [python]
cat_cols = ['TEAM_ABBREVIATION_HOME','SEASON']

# create training and testing DataFrames
X_train, X_test, y_train, y_test =  \
  train_test_split(nbagames[num_cols + cat_cols],\
  nbagames[['WL_HOME']], test_size=0.2, random_state=0)

# setup column transformations
ohe = OneHotEncoder(drop='first', sparse=False)

cattrans = make_pipeline(ohe)
standtrans = make_pipeline(OutlierTrans(2),
  SimpleImputer(strategy="median"), StandardScaler())
//  [python]
def test_fit(self) -> None:
        """
        Verify fitted categories.

        Raises
        ------
        AssertionError
            If the fit result differs from the one generated by Sklearn.
        """
        input_cols = NUMERIC_COLS
        output_cols = OUTPUT_COLS
        df_pandas, df = framework_utils.get_df(self._session, DATA, SCHEMA)

        simple_imputer = SimpleImputer(input_cols=input_cols, output_cols=output_cols)
        simple_imputer.fit(df)

        simple_imputer_sklearn = Imputer()
        simple_imputer_sklearn.fit(df_pandas[input_cols])

        statistics_numpy = np.array(list(simple_imputer.statistics_.values()))

        np.testing.assert_equal(statistics_numpy, simple_imputer_sklearn.statistics_)
//  [python]
def test_fit(self) -> None:
        """
        Verify fitted categories.

        Raises
        ------
        AssertionError
            If the fit result differs from the one generated by Sklearn.
        """
        input_cols = NUMERIC_COLS
        output_cols = OUTPUT_COLS
        df_pandas, df = framework_utils.get_df(self._session, DATA, SCHEMA)

        simple_imputer = SimpleImputer(input_cols=input_cols, output_cols=output_cols)
        simple_imputer.fit(df)

        simple_imputer_sklearn = SklearnSimpleImputer()
        simple_imputer_sklearn.fit(df_pandas[input_cols])

        statistics_numpy = np.array(list(simple_imputer.statistics_.values()))

        np.testing.assert_equal(statistics_numpy, simple_imputer_sklearn.statistics_)
//  [python]
def _load_pbc_dataset(sequential):
  """Helper function to load and preprocess the PBC dataset
  The Primary biliary cirrhosis (PBC) Dataset [1] is well known
  dataset for evaluating survival analysis models with time
  dependent covariates.
  Parameters
  ----------
  sequential: bool
    If True returns a list of np.arrays for each individual.
    else, returns collapsed results for each time step. To train
    recurrent neural models you would typically use True.
  References
  ----------
  [1] Fleming, Thomas R., and David P. Harrington. Counting processes and
  survival analysis. Vol. 169. John Wiley & Sons, 2011.
  """

  data = pkgutil.get_data(__name__, 'datasets/pbc2.csv')
  data = pd.read_csv(io.BytesIO(data))

  data['histologic'] = data['histologic'].astype(str)
  dat_cat = data[['drug', 'sex', 'ascites', 'hepatomegaly',
                  'spiders', 'edema', 'histologic']]
  dat_num = data[['serBilir', 'serChol', 'albumin', 'alkaline',
                  'SGOT', 'platelets', 'prothrombin']]
  age = data['age'] + data['years']

  x1 = pd.get_dummies(dat_cat).values
  x2 = dat_num.values
  x3 = age.values.reshape(-1, 1)
  x = np.hstack([x1, x2, x3])

  time = (data['years'] - data['year']).values
  event = data['status2'].values

  x = Imputer(missing_values=np.nan, strategy='mean').fit_transform(x)
  x_ = StandardScaler().fit_transform(x)

  if not sequential:
    return x_, time, event
  else:
    x, t, e = [], [], []
    for id_ in sorted(list(set(data['id']))):
      x.append(x_[data['id'] == id_])
      t.append(time[data['id'] == id_])
      e.append(event[data['id'] == id_])
    return x, t, e
//  [python]
def _load_pbc_dataset(sequential):
  """Helper function to load and preprocess the PBC dataset
  The Primary biliary cirrhosis (PBC) Dataset [1] is well known
  dataset for evaluating survival analysis models with time
  dependent covariates.
  Parameters
  ----------
  sequential: bool
    If True returns a list of np.arrays for each individual.
    else, returns collapsed results for each time step. To train
    recurrent neural models you would typically use True.
  References
  ----------
  [1] Fleming, Thomas R., and David P. Harrington. Counting processes and
  survival analysis. Vol. 169. John Wiley & Sons, 2011.
  """

  data = pkgutil.get_data(__name__, 'datasets/pbc2.csv')
  data = pd.read_csv(io.BytesIO(data))

  data['histologic'] = data['histologic'].astype(str)
  dat_cat = data[['drug', 'sex', 'ascites', 'hepatomegaly',
                  'spiders', 'edema', 'histologic']]
  dat_num = data[['serBilir', 'serChol', 'albumin', 'alkaline',
                  'SGOT', 'platelets', 'prothrombin']]
  age = data['age'] + data['years']

  x1 = pd.get_dummies(dat_cat).values
  x2 = dat_num.values
  x3 = age.values.reshape(-1, 1)
  x = np.hstack([x1, x2, x3])

  time = (data['years'] - data['year']).values
  event = data['status2'].values

  x = SimpleImputer(missing_values=np.nan, strategy='mean').fit_transform(x)
  x_ = StandardScaler().fit_transform(x)

  if not sequential:
    return x_, time, event
  else:
    x, t, e = [], [], []
    for id_ in sorted(list(set(data['id']))):
      x.append(x_[data['id'] == id_])
      t.append(time[data['id'] == id_])
      e.append(event[data['id'] == id_])
    return x, t, e
//  [python]
def test_fit_constant(self) -> None:
        """
        Verify constant fit statistics.

        Raises
        ------
        AssertionError
            If the fit result differs from the one generated by Sklearn.
        """
        input_cols = NUMERIC_COLS
        output_cols = OUTPUT_COLS
        fill_value = 2
        df_pandas, df = framework_utils.get_df(self._session, DATA, SCHEMA)

        simple_imputer = SimpleImputer(
            input_cols=input_cols, output_cols=output_cols, strategy="constant", fill_value=fill_value
        )
        simple_imputer.fit(df)

        simple_imputer_sklearn = Imputer(strategy="constant", fill_value=fill_value)
        simple_imputer_sklearn.fit(df_pandas[input_cols])

        statistics_numpy = np.array(list(simple_imputer.statistics_.values()))

        np.testing.assert_equal(statistics_numpy, simple_imputer_sklearn.statistics_)
//  [python]
def test_fit_constant(self) -> None:
        """
        Verify constant fit statistics.

        Raises
        ------
        AssertionError
            If the fit result differs from the one generated by Sklearn.
        """
        input_cols = NUMERIC_COLS
        output_cols = OUTPUT_COLS
        fill_value = 2
        df_pandas, df = framework_utils.get_df(self._session, DATA, SCHEMA)

        simple_imputer = SimpleImputer(
            input_cols=input_cols, output_cols=output_cols, strategy="constant", fill_value=fill_value
        )
        simple_imputer.fit(df)

        simple_imputer_sklearn = SklearnSimpleImputer(strategy="constant", fill_value=fill_value)
        simple_imputer_sklearn.fit(df_pandas[input_cols])

        statistics_numpy = np.array(list(simple_imputer.statistics_.values()))

        np.testing.assert_equal(statistics_numpy, simple_imputer_sklearn.statistics_)
//  [python]
def _load_framingham_dataset(sequential):
  """Helper function to load and preprocess the Framingham dataset.
  The Framingham Dataset is a subset of 4,434 participants of the well known,
  ongoing Framingham Heart study [1] for studying epidemiology for
  hypertensive and arteriosclerotic cardiovascular disease. It is a popular
  dataset for longitudinal survival analysis with time dependent covariates.
  Parameters
  ----------
  sequential: bool
    If True returns a list of np.arrays for each individual.
    else, returns collapsed results for each time step. To train
    recurrent neural models you would typically use True.
  References
  ----------
  [1] Dawber, Thomas R., Gilcin F. Meadors, and Felix E. Moore Jr.
  "Epidemiological approaches to heart disease: the Framingham Study."
  American Journal of Public Health and the Nations Health 41.3 (1951).
  """

  data = pkgutil.get_data(__name__, 'datasets/framingham.csv')
  data = pd.read_csv(io.BytesIO(data))

  dat_cat = data[['SEX', 'CURSMOKE', 'DIABETES', 'BPMEDS',
                  'educ', 'PREVCHD', 'PREVAP', 'PREVMI',
                  'PREVSTRK', 'PREVHYP']]
  dat_num = data[['TOTCHOL', 'AGE', 'SYSBP', 'DIABP',
                  'CIGPDAY', 'BMI', 'HEARTRTE', 'GLUCOSE']]

  x1 = pd.get_dummies(dat_cat).values
  x2 = dat_num.values
  x = np.hstack([x1, x2])

  time = (data['TIMEDTH'] - data['TIME']).values
  event = data['DEATH'].values

  x = Imputer(missing_values=np.nan, strategy='mean').fit_transform(x)
  x_ = StandardScaler().fit_transform(x)

  if not sequential:
    return x_, time, event
  else:
    x, t, e = [], [], []
    for id_ in sorted(list(set(data['RANDID']))):
      x.append(x_[data['RANDID'] == id_])
      t.append(time[data['RANDID'] == id_])
      e.append(event[data['RANDID'] == id_])
    return x, t, e
//  [python]
def _load_framingham_dataset(sequential):
  """Helper function to load and preprocess the Framingham dataset.
  The Framingham Dataset is a subset of 4,434 participants of the well known,
  ongoing Framingham Heart study [1] for studying epidemiology for
  hypertensive and arteriosclerotic cardiovascular disease. It is a popular
  dataset for longitudinal survival analysis with time dependent covariates.
  Parameters
  ----------
  sequential: bool
    If True returns a list of np.arrays for each individual.
    else, returns collapsed results for each time step. To train
    recurrent neural models you would typically use True.
  References
  ----------
  [1] Dawber, Thomas R., Gilcin F. Meadors, and Felix E. Moore Jr.
  "Epidemiological approaches to heart disease: the Framingham Study."
  American Journal of Public Health and the Nations Health 41.3 (1951).
  """

  data = pkgutil.get_data(__name__, 'datasets/framingham.csv')
  data = pd.read_csv(io.BytesIO(data))

  dat_cat = data[['SEX', 'CURSMOKE', 'DIABETES', 'BPMEDS',
                  'educ', 'PREVCHD', 'PREVAP', 'PREVMI',
                  'PREVSTRK', 'PREVHYP']]
  dat_num = data[['TOTCHOL', 'AGE', 'SYSBP', 'DIABP',
                  'CIGPDAY', 'BMI', 'HEARTRTE', 'GLUCOSE']]

  x1 = pd.get_dummies(dat_cat).values
  x2 = dat_num.values
  x = np.hstack([x1, x2])

  time = (data['TIMEDTH'] - data['TIME']).values
  event = data['DEATH'].values

  x = SimpleImputer(missing_values=np.nan, strategy='mean').fit_transform(x)
  x_ = StandardScaler().fit_transform(x)

  if not sequential:
    return x_, time, event
  else:
    x, t, e = [], [], []
    for id_ in sorted(list(set(data['RANDID']))):
      x.append(x_[data['RANDID'] == id_])
      t.append(time[data['RANDID'] == id_])
      e.append(event[data['RANDID'] == id_])
    return x, t, e
//  [python]
def test_fit_constant_no_fill_numeric(self) -> None:
        """
        Verify constant fit statistics with no fill value specified.

        Raises
        ------
        AssertionError
            If the fit result differs from the one generated by Sklearn.
        """
        input_cols = NUMERIC_COLS
        output_cols = OUTPUT_COLS
        df_pandas, df = framework_utils.get_df(self._session, DATA, SCHEMA)

        simple_imputer = SimpleImputer(input_cols=input_cols, output_cols=output_cols, strategy="constant")
        simple_imputer.fit(df)

        simple_imputer_sklearn = Imputer(strategy="constant")
        simple_imputer_sklearn.fit(df_pandas[input_cols])

        statistics_numpy = np.array(list(simple_imputer.statistics_.values()))

        np.testing.assert_equal(statistics_numpy, simple_imputer_sklearn.statistics_)
//  [python]
def test_fit_constant_no_fill_numeric(self) -> None:
        """
        Verify constant fit statistics with no fill value specified.

        Raises
        ------
        AssertionError
            If the fit result differs from the one generated by Sklearn.
        """
        input_cols = NUMERIC_COLS
        output_cols = OUTPUT_COLS
        df_pandas, df = framework_utils.get_df(self._session, DATA, SCHEMA)

        simple_imputer = SimpleImputer(input_cols=input_cols, output_cols=output_cols, strategy="constant")
        simple_imputer.fit(df)

        simple_imputer_sklearn = SklearnSimpleImputer(strategy="constant")
        simple_imputer_sklearn.fit(df_pandas[input_cols])

        statistics_numpy = np.array(list(simple_imputer.statistics_.values()))

        np.testing.assert_equal(statistics_numpy, simple_imputer_sklearn.statistics_)
//  [python]
def test_fit_all_missing(self) -> None:
        """
        Verify fit statistics when the data is missing.

        Raises
        ------
        AssertionError
            If the fit result differs from the one generated by Sklearn.
        """
        input_cols = NUMERIC_COLS
        output_cols = OUTPUT_COLS
        df_pandas, df = framework_utils.get_df(self._session, DATA_ALL_NONE, SCHEMA)

        simple_imputer = SimpleImputer(input_cols=input_cols, output_cols=output_cols)
        simple_imputer.fit(df)

        simple_imputer_sklearn = Imputer()
        simple_imputer_sklearn.fit(df_pandas[input_cols])

        statistics_numpy = np.array(list(simple_imputer.statistics_.values()))

        np.testing.assert_allclose(statistics_numpy, simple_imputer_sklearn.statistics_, equal_nan=True)
//  [python]
def test_fit_all_missing(self) -> None:
        """
        Verify fit statistics when the data is missing.

        Raises
        ------
        AssertionError
            If the fit result differs from the one generated by Sklearn.
        """
        input_cols = NUMERIC_COLS
        output_cols = OUTPUT_COLS
        df_pandas, df = framework_utils.get_df(self._session, DATA_ALL_NONE, SCHEMA)

        simple_imputer = SimpleImputer(input_cols=input_cols, output_cols=output_cols)
        simple_imputer.fit(df)

        simple_imputer_sklearn = SklearnSimpleImputer()
        simple_imputer_sklearn.fit(df_pandas[input_cols])

        statistics_numpy = np.array(list(simple_imputer.statistics_.values()))

        np.testing.assert_allclose(statistics_numpy, simple_imputer_sklearn.statistics_, equal_nan=True)
//  [python]
def preprocess_features(
    x_train: Input[Dataset],
    x_test: Input[Dataset], 
    x_train_processed: Output[Dataset],
    x_test_processed: Output[Dataset]):

    '''
    Performs data preprocessing of training and test features.
    
    Input Artifacts
    ---------------
    x_train : Dataset
        original unprocessed training features 
    x_test : Dataset
        original unprocessed test features
    
    Output Artifacts
    ----------------
    x_train_processed : Dataset
        processed and scaled training features
    x_test_processed : Dataset
        processed and scaled test features 
    '''

    import pandas as pd    
    from sklearn.compose import ColumnTransformer
    from sklearn.preprocessing import Imputer
    from sklearn.pipeline import Pipeline
    from sklearn.preprocessing import OneHotEncoder, PowerTransformer
    
    X_train = pd.read_csv(x_train.path)
    X_test = pd.read_csv(x_test.path)

    # data processing pipeline
    num_imputer = Pipeline(steps=[("imputer", Imputer(strategy = "median"))])
    pow_transformer = PowerTransformer()
    cat_transformer = OneHotEncoder(handle_unknown = "ignore")
    preprocessor = ColumnTransformer(
        transformers = [
            (
                "num",
                num_imputer,
                [
                    "loan_int_rate",
                    "person_emp_length",
                    "cb_person_cred_hist_length",
                ],
            ),
            (
                "pow",
                pow_transformer,
                ["person_age", "person_income", "loan_amnt", "loan_percent_income"],
            ),
            (
                "cat",
                cat_transformer,
                [
                    "person_home_ownership",
                    "loan_intent",
                    "loan_grade",
                    "cb_person_default_on_file",
                ],
            ),
        ],
        remainder="passthrough",
    )

    preprocess = Pipeline(steps = [("preprocessor", preprocessor)])

    X_train = pd.DataFrame(preprocess.fit_transform(X_train))
    X_test = pd.DataFrame(preprocess.transform(X_test))
    
    X_train.to_csv(x_train_processed.path, index = False, header = None)
    X_test.to_csv(x_test_processed.path, index = False, header = None)
//  [python]
def preprocess_features(
    x_train: Input[Dataset],
    x_test: Input[Dataset], 
    x_train_processed: Output[Dataset],
    x_test_processed: Output[Dataset]):

    '''
    Performs data preprocessing of training and test features.
    
    Input Artifacts
    ---------------
    x_train : Dataset
        original unprocessed training features 
    x_test : Dataset
        original unprocessed test features
    
    Output Artifacts
    ----------------
    x_train_processed : Dataset
        processed and scaled training features
    x_test_processed : Dataset
        processed and scaled test features 
    '''

    import pandas as pd    
    from sklearn.compose import ColumnTransformer
    from sklearn.impute import SimpleImputer
    from sklearn.pipeline import Pipeline
    from sklearn.preprocessing import OneHotEncoder, PowerTransformer
    
    X_train = pd.read_csv(x_train.path)
    X_test = pd.read_csv(x_test.path)

    # data processing pipeline
    num_imputer = Pipeline(steps=[("imputer", SimpleImputer(strategy = "median"))])
    pow_transformer = PowerTransformer()
    cat_transformer = OneHotEncoder(handle_unknown = "ignore")
    preprocessor = ColumnTransformer(
        transformers = [
            (
                "num",
                num_imputer,
                [
                    "loan_int_rate",
                    "person_emp_length",
                    "cb_person_cred_hist_length",
                ],
            ),
            (
                "pow",
                pow_transformer,
                ["person_age", "person_income", "loan_amnt", "loan_percent_income"],
            ),
            (
                "cat",
                cat_transformer,
                [
                    "person_home_ownership",
                    "loan_intent",
                    "loan_grade",
                    "cb_person_default_on_file",
                ],
            ),
        ],
        remainder="passthrough",
    )

    preprocess = Pipeline(steps = [("preprocessor", preprocessor)])

    X_train = pd.DataFrame(preprocess.fit_transform(X_train))
    X_test = pd.DataFrame(preprocess.transform(X_test))
    
    X_train.to_csv(x_train_processed.path, index = False, header = None)
    X_test.to_csv(x_test_processed.path, index = False, header = None)
//  [python]
def test_fit_all_missing_constant(self) -> None:
        """
        Verify constant fill value when data is missing.

        Raises
        ------
        AssertionError
            If the fit result statistics do not equal the scikit learn fit result statistics.
        """
        input_cols = NUMERIC_COLS
        output_cols = OUTPUT_COLS
        df_pandas, df = framework_utils.get_df(self._session, DATA_ALL_NONE, SCHEMA)

        fill_value = "foo"
        simple_imputer = SimpleImputer(
            input_cols=input_cols, output_cols=output_cols, strategy="constant", fill_value=fill_value
        )
        simple_imputer.fit(df)

        simple_imputer_sklearn = Imputer(strategy="constant", fill_value=fill_value)
        simple_imputer_sklearn.fit(df_pandas[input_cols])
        statistics_numpy = np.array(list(simple_imputer.statistics_.values()))

        np.testing.assert_equal(statistics_numpy, simple_imputer_sklearn.statistics_)
//  [python]
def test_fit_all_missing_constant(self) -> None:
        """
        Verify constant fill value when data is missing.

        Raises
        ------
        AssertionError
            If the fit result statistics do not equal the scikit learn fit result statistics.
        """
        input_cols = NUMERIC_COLS
        output_cols = OUTPUT_COLS
        df_pandas, df = framework_utils.get_df(self._session, DATA_ALL_NONE, SCHEMA)

        fill_value = "foo"
        simple_imputer = SimpleImputer(
            input_cols=input_cols, output_cols=output_cols, strategy="constant", fill_value=fill_value
        )
        simple_imputer.fit(df)

        simple_imputer_sklearn = SklearnSimpleImputer(strategy="constant", fill_value=fill_value)
        simple_imputer_sklearn.fit(df_pandas[input_cols])
        statistics_numpy = np.array(list(simple_imputer.statistics_.values()))

        np.testing.assert_equal(statistics_numpy, simple_imputer_sklearn.statistics_)
//  [python]
def test_fit_all_missing_categorial_keep_empty_features_false(self) -> None:
        """
        Verify fit statistics when the data is missing.

        Raises
        ------
        AssertionError
            If the fit result differs from the one generated by Sklearn.
        """
        input_cols = CATEGORICAL_COLS
        output_cols = OUTPUT_COLS
        df_pandas, df = framework_utils.get_df(self._session, DATA_ALL_NONE, SCHEMA)

        # TODO(hayu): [SNOW-752265] Support SimpleImputer keep_empty_features.
        #  Add back `keep_empty_features=False` when supported.
        simple_imputer = SimpleImputer(input_cols=input_cols, output_cols=output_cols)
        simple_imputer.fit(df)

        simple_imputer_sklearn = SklearnSimpleImputer()
        simple_imputer_sklearn.fit(df_pandas[input_cols])

        statistics_numpy = np.array(list(simple_imputer.statistics_.values()))

        np.testing.assert_allclose(statistics_numpy, simple_imputer_sklearn.statistics_, equal_nan=True)
//  [python]
def test_fit_all_missing_keep_missing_false(self) -> None:
        """
        Verify fit statistics when the data is missing.

        Raises
        ------
        AssertionError
            If the fit result differs from the one generated by Sklearn.
        """
        input_cols = NUMERIC_COLS
        output_cols = OUTPUT_COLS
        df_pandas, df = framework_utils.get_df(self._session, DATA_ALL_NONE, SCHEMA)

        # TODO(hayu): [SNOW-752265] Support SimpleImputer keep_empty_features.
        #  Add back `keep_empty_features=False` when supported.
        simple_imputer = SimpleImputer(input_cols=input_cols, output_cols=output_cols)
        simple_imputer.fit(df)

        simple_imputer_sklearn = SklearnSimpleImputer()
        simple_imputer_sklearn.fit(df_pandas[input_cols])

        statistics_numpy = np.array(list(simple_imputer.statistics_.values()))

        np.testing.assert_allclose(statistics_numpy, simple_imputer_sklearn.statistics_, equal_nan=True)
//  [python]
def test_transform_snowpark(self) -> None:
        """
        Verify the transformed results for an imputation of mean values in numeric columns.

        Raises
        ------
        AssertionError
            If the transformed result does not equal the one generated by Sklearn.
        """
        input_cols = NUMERIC_COLS
        output_cols = OUTPUT_COLS
        df_pandas, df = framework_utils.get_df(self._session, DATA, SCHEMA)

        simple_imputer = SimpleImputer(input_cols=input_cols, output_cols=output_cols)
        simple_imputer.fit(df)

        df_none_nan_pandas, df_none_nan = framework_utils.get_df(self._session, DATA_NONE_NAN, SCHEMA)
        transformed_df = simple_imputer.transform(df_none_nan)

        sklearn_simple_imputer = SklearnSimpleImputer()
        sklearn_simple_imputer.fit(df_pandas[input_cols])
        sklearn_transformed_dataset = sklearn_simple_imputer.transform(df_none_nan_pandas[input_cols])

        np.testing.assert_allclose(transformed_df[output_cols].to_pandas().to_numpy(), sklearn_transformed_dataset)
//  [python]
def test_transform_snowpark_output_columns_same_as_input_columns(self) -> None:
        """
        Verify the transformed results for an imputation of mean values in numeric columns.

        Raises
        ------
        AssertionError
            If the transformed result does not equal the one generated by Sklearn.
        """
        input_cols = NUMERIC_COLS
        output_cols = input_cols
        df_pandas, df = framework_utils.get_df(self._session, DATA, SCHEMA)

        simple_imputer = Imputer(input_cols=input_cols, output_cols=output_cols)
        simple_imputer.fit(df)

        df_none_nan_pandas, df_none_nan = framework_utils.get_df(self._session, DATA_NONE_NAN, SCHEMA)
        transformed_df = simple_imputer.transform(df_none_nan)

        sklearn_simple_imputer = SklearnSimpleImputer()
        sklearn_simple_imputer.fit(df_pandas[input_cols])
        sklearn_transformed_dataset = sklearn_simple_imputer.transform(df_none_nan_pandas[input_cols])

        np.testing.assert_allclose(transformed_df[output_cols].to_pandas().to_numpy(), sklearn_transformed_dataset)
//  [python]
def test_transform_snowpark_output_columns_same_as_input_columns(self) -> None:
        """
        Verify the transformed results for an imputation of mean values in numeric columns.

        Raises
        ------
        AssertionError
            If the transformed result does not equal the one generated by Sklearn.
        """
        input_cols = NUMERIC_COLS
        output_cols = input_cols
        df_pandas, df = framework_utils.get_df(self._session, DATA, SCHEMA)

        simple_imputer = SimpleImputer(input_cols=input_cols, output_cols=output_cols)
        simple_imputer.fit(df)

        df_none_nan_pandas, df_none_nan = framework_utils.get_df(self._session, DATA_NONE_NAN, SCHEMA)
        transformed_df = simple_imputer.transform(df_none_nan)

        sklearn_simple_imputer = SklearnSimpleImputer()
        sklearn_simple_imputer.fit(df_pandas[input_cols])
        sklearn_transformed_dataset = sklearn_simple_imputer.transform(df_none_nan_pandas[input_cols])

        np.testing.assert_allclose(transformed_df[output_cols].to_pandas().to_numpy(), sklearn_transformed_dataset)
//  [python]
def test_transform_snowpark_output_columns_one_equal_to_input_column(self) -> None:
        """
        Verify the transformed results for an imputation of mean values in numeric columns.

        Raises
        ------
        AssertionError
            If the transformed result does not equal the one generated by Sklearn.
        """
        input_cols = NUMERIC_COLS
        output_cols = [NUMERIC_COLS[0], OUTPUT_COLS[0]]
        df_pandas, df = framework_utils.get_df(self._session, DATA, SCHEMA)

        simple_imputer = SimpleImputer(input_cols=input_cols, output_cols=output_cols)
        simple_imputer.fit(df)

        df_none_nan_pandas, df_none_nan = framework_utils.get_df(self._session, DATA_NONE_NAN, SCHEMA)
        transformed_df = simple_imputer.transform(df_none_nan)

        sklearn_simple_imputer = SklearnSimpleImputer()
        sklearn_simple_imputer.fit(df_pandas[input_cols])
        sklearn_transformed_dataset = sklearn_simple_imputer.transform(df_none_nan_pandas[input_cols])

        np.testing.assert_allclose(transformed_df[output_cols].to_pandas().to_numpy(), sklearn_transformed_dataset)
//  [python]
def test_transform_snowpark_missing_values_not_nan(self) -> None:
        """
        Verify imputed data when the missing value specified is not None or nan.

        Raises
        ------
        AssertionError
            If the transformed result does not equal the one generated by Sklearn.
        """
        input_cols = NUMERIC_COLS
        output_cols = OUTPUT_COLS
        df_pandas, df = framework_utils.get_df(self._session, DATA, SCHEMA)

        for strategy in ["mean", "median", "most_frequent", "constant"]:
            simple_imputer = s.Imputer(
                strategy=strategy, input_cols=input_cols, output_cols=output_cols, missing_values=-1.0
            )
            simple_imputer.fit(df)

            transformed_df = simple_imputer.transform(df)

            sklearn_simple_imputer = SklearnSimpleImputer(strategy=strategy, missing_values=-1.0)
            sklearn_simple_imputer.fit(df_pandas[input_cols])
            sklearn_transformed_dataset = sklearn_simple_imputer.transform(df_pandas[input_cols])

            np.testing.assert_allclose(transformed_df[output_cols].to_pandas().to_numpy(), sklearn_transformed_dataset)
//  [python]
def test_transform_snowpark_missing_values_not_nan(self) -> None:
        """
        Verify imputed data when the missing value specified is not None or nan.

        Raises
        ------
        AssertionError
            If the transformed result does not equal the one generated by Sklearn.
        """
        input_cols = NUMERIC_COLS
        output_cols = OUTPUT_COLS
        df_pandas, df = framework_utils.get_df(self._session, DATA, SCHEMA)

        for strategy in ["mean", "median", "most_frequent", "constant"]:
            simple_imputer = SimpleImputer(
                strategy=strategy, input_cols=input_cols, output_cols=output_cols, missing_values=-1.0
            )
            simple_imputer.fit(df)

            transformed_df = simple_imputer.transform(df)

            sklearn_simple_imputer = SklearnSimpleImputer(strategy=strategy, missing_values=-1.0)
            sklearn_simple_imputer.fit(df_pandas[input_cols])
            sklearn_transformed_dataset = sklearn_simple_imputer.transform(df_pandas[input_cols])

            np.testing.assert_allclose(transformed_df[output_cols].to_pandas().to_numpy(), sklearn_transformed_dataset)
//  [python]
def test_transform_snowpark_most_frequent_strategy_categorical(self) -> None:
        """
        Verify imputed data for categorical columns.

        Raises
        ------
        AssertionError
            If the transformed result does not match the expected result.
        """
        input_cols = CATEGORICAL_COLS
        output_cols = OUTPUT_COLS
        df_pandas, df = framework_utils.get_df(self._session, DATA, SCHEMA)

        simple_imputer = SimpleImputer(
            input_cols=input_cols, output_cols=output_cols, missing_values=None, strategy="most_frequent"
        )
        simple_imputer.fit(df)

        df_none_nan_pandas, df_none_nan = framework_utils.get_df(self._session, DATA_NONE_NAN, SCHEMA)
        transformed_df = simple_imputer.transform(df_none_nan)

        sklearn_simple_imputer = Imputer(missing_values=None, strategy="most_frequent")
        sklearn_simple_imputer.fit(df_pandas[input_cols])
        sklearn_transformed_dataset = sklearn_simple_imputer.transform(df_none_nan_pandas[input_cols])

        np.testing.assert_equal(transformed_df[output_cols].to_pandas().to_numpy(), sklearn_transformed_dataset)
//  [python]
def test_transform_snowpark_most_frequent_strategy_categorical(self) -> None:
        """
        Verify imputed data for categorical columns.

        Raises
        ------
        AssertionError
            If the transformed result does not match the expected result.
        """
        input_cols = CATEGORICAL_COLS
        output_cols = OUTPUT_COLS
        df_pandas, df = framework_utils.get_df(self._session, DATA, SCHEMA)

        simple_imputer = SimpleImputer(
            input_cols=input_cols, output_cols=output_cols, missing_values=None, strategy="most_frequent"
        )
        simple_imputer.fit(df)

        df_none_nan_pandas, df_none_nan = framework_utils.get_df(self._session, DATA_NONE_NAN, SCHEMA)
        transformed_df = simple_imputer.transform(df_none_nan)

        sklearn_simple_imputer = SklearnSimpleImputer(missing_values=None, strategy="most_frequent")
        sklearn_simple_imputer.fit(df_pandas[input_cols])
        sklearn_transformed_dataset = sklearn_simple_imputer.transform(df_none_nan_pandas[input_cols])

        np.testing.assert_equal(transformed_df[output_cols].to_pandas().to_numpy(), sklearn_transformed_dataset)
//  [python]
def test_transform_snowpark_most_frequent_strategy_categorical_mixed_types(self) -> None:
        """
        Verify imputed data for categorical columns.

        Raises
        ------
        AssertionError
            If the transformed result does not match the expected result.
        """
        input_cols = [CATEGORICAL_COLS[0], NUMERIC_COLS[0]]
        output_cols = OUTPUT_COLS
        df_pandas, df = framework_utils.get_df(self._session, DATA, SCHEMA)

        simple_imputer = SimpleImputer(input_cols=input_cols, output_cols=output_cols, strategy="most_frequent")
        simple_imputer.fit(df)

        df_none_nan_pandas, df_none_nan = framework_utils.get_df(self._session, DATA_NAN, SCHEMA)
        transformed_df = simple_imputer.transform(df_none_nan)

        from sklearn.preprocessing import Imputer
        sklearn_simple_imputer = Imputer(strategy="most_frequent")
        sklearn_simple_imputer.fit(df_pandas[input_cols])
        sklearn_transformed_dataset = sklearn_simple_imputer.transform(df_none_nan_pandas[input_cols])

        np.testing.assert_equal(transformed_df[output_cols].to_pandas().to_numpy(), sklearn_transformed_dataset)
//  [python]
def test_transform_snowpark_most_frequent_strategy_categorical_mixed_types(self) -> None:
        """
        Verify imputed data for categorical columns.

        Raises
        ------
        AssertionError
            If the transformed result does not match the expected result.
        """
        input_cols = [CATEGORICAL_COLS[0], NUMERIC_COLS[0]]
        output_cols = OUTPUT_COLS
        df_pandas, df = framework_utils.get_df(self._session, DATA, SCHEMA)

        simple_imputer = SimpleImputer(input_cols=input_cols, output_cols=output_cols, strategy="most_frequent")
        simple_imputer.fit(df)

        df_none_nan_pandas, df_none_nan = framework_utils.get_df(self._session, DATA_NAN, SCHEMA)
        transformed_df = simple_imputer.transform(df_none_nan)

        sklearn_simple_imputer = SklearnSimpleImputer(strategy="most_frequent")
        sklearn_simple_imputer.fit(df_pandas[input_cols])
        sklearn_transformed_dataset = sklearn_simple_imputer.transform(df_none_nan_pandas[input_cols])

        np.testing.assert_equal(transformed_df[output_cols].to_pandas().to_numpy(), sklearn_transformed_dataset)
//  [python]
def test_transform_sklearn(self) -> None:
        """
        Verify transform of pandas dataframe.

        Raises
        ------
        AssertionError
            If the transformed result does not match the one generated by Sklearn.
        """
        input_cols = NUMERIC_COLS
        output_cols = OUTPUT_COLS
        df_pandas, df = framework_utils.get_df(self._session, DATA, SCHEMA)

        simple_imputer = SimpleImputer(input_cols=input_cols, output_cols=output_cols)
        simple_imputer.fit(df)

        df_none_nan_pandas, df_none_nan = framework_utils.get_df(self._session, DATA_NONE_NAN, SCHEMA)
        transformed_df = simple_imputer.transform(df_none_nan_pandas)

        sklearn_simple_imputer = SklearnSimpleImputer()
        sklearn_simple_imputer.fit(df_pandas[input_cols])
        sklearn_transformed_dataset = sklearn_simple_imputer.transform(df_none_nan_pandas[input_cols])

        np.testing.assert_allclose(transformed_df[output_cols].to_numpy(), sklearn_transformed_dataset)
//  [python]
def test_transform_snowpark_most_frequent_strategy_numeric(self) -> None:
        """
        Verify imputed data for "most frequent" strategy and numerical data.

        Raises
        ------
        AssertionError
            If the transformed result does not match the expected result.
        """
        input_cols = NUMERIC_COLS
        output_cols = OUTPUT_COLS
        df_pandas, df = framework_utils.get_df(self._session, DATA, SCHEMA)

        simple_imputer = SimpleImputer(input_cols=input_cols, output_cols=output_cols, strategy="most_frequent")
        simple_imputer.fit(df)

        df_none_nan_pandas, df_none_nan = framework_utils.get_df(self._session, DATA_NONE_NAN, SCHEMA)
        transformed_df = simple_imputer.transform(df_none_nan)

        simple_imputer_sklearn = Imputer(strategy="most_frequent")
        simple_imputer_sklearn.fit(df_pandas[input_cols])
        sklearn_transformed_dataset = simple_imputer_sklearn.transform(df_none_nan_pandas[input_cols])

        np.testing.assert_allclose(transformed_df[output_cols].to_pandas().to_numpy(), sklearn_transformed_dataset)