jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def make_classifier_pipeline(X: pd.DataFrame, num_labels: int) -> Pipeline:
    """
    Make the classifier pipeline with the required preprocessor steps and estimator in the end.

    Parameters
    ----------
    X
        X containing all the required features for training
    num_labels
        number of classification labels

    Returns
    -------
    Pipeline
        Classifier pipeline with preprocessor and estimator
    """
    numerics = ["int16", "int32", "int64", "float16", "float32", "float64"]
    num_features = list(X.dropna(axis=1, how="all").select_dtypes(include=numerics).columns)

    # This example model only uses numeric features and drops the rest
    num_transformer = Pipeline(
        steps=[("imputer", Imputer(strategy="median")), ("scaler", StandardScaler())]
    )

    preprocessor = ColumnTransformer(transformers=[("num", num_transformer, num_features)])

    # create model
    estimator = (
        create_binary_classification_model()
        if num_labels == 2
        else create_multiclass_classification_model(num_labels)
    )

    # pipeline with preprocessor and estimator bundled
    classifier_pipeline = Pipeline(steps=[("preprocessor", preprocessor), ("estimator", estimator)])
    return classifier_pipeline
//  [python]
def make_classifier_pipeline(X: pd.DataFrame, num_labels: int) -> Pipeline:
    """
    Make the classifier pipeline with the required preprocessor steps and estimator in the end.

    Parameters
    ----------
    X
        X containing all the required features for training
    num_labels
        number of classification labels

    Returns
    -------
    Pipeline
        Classifier pipeline with preprocessor and estimator
    """
    numerics = ["int16", "int32", "int64", "float16", "float32", "float64"]
    num_features = list(X.dropna(axis=1, how="all").select_dtypes(include=numerics).columns)

    # This example model only uses numeric features and drops the rest
    num_transformer = Pipeline(
        steps=[("imputer", SimpleImputer(strategy="median")), ("scaler", StandardScaler())]
    )

    preprocessor = ColumnTransformer(transformers=[("num", num_transformer, num_features)])

    # create model
    estimator = (
        create_binary_classification_model()
        if num_labels == 2
        else create_multiclass_classification_model(num_labels)
    )

    # pipeline with preprocessor and estimator bundled
    classifier_pipeline = Pipeline(steps=[("preprocessor", preprocessor), ("estimator", estimator)])
    return classifier_pipeline
//  [python]
def prepare_digits_all():
    data = sklearn.datasets.fetch_mldata('MNIST original', data_home='../data')
    X = data.data.astype(float)
    y = np.asarray(data.target, dtype=int)
    return X, y
//  [python]
def prepare_digits_all():
    data = sklearn.datasets.fetch_openml('mnist_784', version=1, data_home='../data')
    X = data.data.astype(float)
    y = np.asarray(data.target, dtype=int)
    return X, y
//  [python]
def doit(shuffled):
    mnist = sklearn.datasets.fetch_mldata('MNIST original', data_home=".")
    x, y = mnist.data, mnist.target

    x_train, x_test = x[:N_TRAIN], x[N_TRAIN:]
    y_train, y_test = y[:N_TRAIN], y[N_TRAIN:]
    if shuffled:
        y_train = np.random.permutation(y_train)
    
    result = []
    for model_name, model_class, model_kwargs in MODELS:
        score_train, score_test = [], []
        for n_iterations in NS_ITERATIONS:
            model_obj = model_class(n_estimators=n_iterations, **model_kwargs)
            model_obj.fit(x_train, y_train)
            score_train.append(model_obj.score(x_train, y_train))
            score_test.append(model_obj.score(x_test, y_test))
        result.append((model_name, score_train, score_test))

    return result
//  [python]
def doit(shuffled):
    mnist = sklearn.datasets.fetch_openml('mnist_784', version=1, data_home=".")
    x, y = mnist.data, mnist.target

    x_train, x_test = x[:N_TRAIN], x[N_TRAIN:]
    y_train, y_test = y[:N_TRAIN], y[N_TRAIN:]
    if shuffled:
        y_train = np.random.permutation(y_train)
    
    result = []
    for model_name, model_class, model_kwargs in MODELS:
        score_train, score_test = [], []
        for n_iterations in NS_ITERATIONS:
            model_obj = model_class(n_estimators=n_iterations, **model_kwargs)
            model_obj.fit(x_train, y_train)
            score_train.append(model_obj.score(x_train, y_train))
            score_test.append(model_obj.score(x_test, y_test))
        result.append((model_name, score_train, score_test))

    return result
//  [python]
def standard_transform(df, dtypes, skip: Optional[list] = None, robust=False):
    """Applies standard transformation to the dataset (imputation, centering and scaling), returns transformed data
    and the fitted transformer.
    Numerical data is imputed with mean, centered and scaled by 4 standard deviations.
    Categorical data is imputed with mode. Encoding is not performed in this stage to preserve the same columns.
    If robust is passed as True, will truncate  numerical data before computing statistics.
    [1]From 1997 Wilson, D. Randall; Martinez, Tony R. -
        Improved Heterogeneous Distance Functions https://arxiv.org/pdf/cs/9701101.pdf
    """
    skip = [] if skip is None else skip
    numerical_features = [key for key, value in dtypes.items() if value == 'numerical' and key not in skip]
    categorical_features = [key for key, value in dtypes.items() if value == 'categorical' and key not in skip]
    assert len(numerical_features + categorical_features +
               skip) == len(df.columns), 'the union of dtypes keys with skip should be the same as the df columns'
    if robust:
        numeric_transformer = Pipeline([
            ('imputer', Imputer()),
            ('scaler', RobustScaler(quantile_range=(5.0, 95.0)))])
    else:
        numeric_transformer = NUMERIC_TRANSFORMER
    preprocessor = ColumnTransformer(
        transformers=[  # Numerical vars are scaled by 4sd so that most of the data are fit in the [-1, 1] range
            ('num', Pipeline(numeric_transformer.steps + \
             [('divby4', FunctionTransformer(lambda x: x / 4))]), numerical_features),
            ('cat', Pipeline([('impute', Imputer(strategy='most_frequent'))]), categorical_features)],
        remainder='passthrough')
    new_column_order = numerical_features + categorical_features + skip
    tdf = DataFrame(preprocessor.fit_transform(df), index=df.index, columns=new_column_order)
    return tdf, preprocessor
//  [python]
def standard_transform(df, dtypes, skip: Optional[list] = None, robust=False):
    """Applies standard transformation to the dataset (imputation, centering and scaling), returns transformed data
    and the fitted transformer.
    Numerical data is imputed with mean, centered and scaled by 4 standard deviations.
    Categorical data is imputed with mode. Encoding is not performed in this stage to preserve the same columns.
    If robust is passed as True, will truncate  numerical data before computing statistics.
    [1]From 1997 Wilson, D. Randall; Martinez, Tony R. -
        Improved Heterogeneous Distance Functions https://arxiv.org/pdf/cs/9701101.pdf
    """
    skip = [] if skip is None else skip
    numerical_features = [key for key, value in dtypes.items() if value == 'numerical' and key not in skip]
    categorical_features = [key for key, value in dtypes.items() if value == 'categorical' and key not in skip]
    assert len(numerical_features + categorical_features +
               skip) == len(df.columns), 'the union of dtypes keys with skip should be the same as the df columns'
    if robust:
        numeric_transformer = Pipeline([
            ('imputer', SimpleImputer()),
            ('scaler', RobustScaler(quantile_range=(5.0, 95.0)))])
    else:
        numeric_transformer = NUMERIC_TRANSFORMER
    preprocessor = ColumnTransformer(
        transformers=[  # Numerical vars are scaled by 4sd so that most of the data are fit in the [-1, 1] range
            ('num', Pipeline(numeric_transformer.steps + \
             [('divby4', FunctionTransformer(lambda x: x / 4))]), numerical_features),
            ('cat', Pipeline([('impute', SimpleImputer(strategy='most_frequent'))]), categorical_features)],
        remainder='passthrough')
    new_column_order = numerical_features + categorical_features + skip
    tdf = DataFrame(preprocessor.fit_transform(df), index=df.index, columns=new_column_order)
    return tdf, preprocessor
//  [python]
predictor_name = 'predict_'
version = 'MNIST_example'
extension = '.npz'

print("mnist=Downloading")
mnist = dt.fetch_mldata('MNIST original')
//  [python]
predictor_name = 'predict_'
version = 'MNIST_example'
extension = '.npz'

print("mnist=Downloading")
mnist = fetch_openml('mnist_784')
//  [python]
def verify_explain_model_categorical(self, pass_categoricals=False, verify_same_shape=False):
        headers = ["symboling", "normalized_losses", "make", "fuel_type", "aspiration",
                   "num_doors", "body_style", "drive_wheels", "engine_location",
                   "wheel_base", "length", "width", "height", "curb_weight",
                   "engine_type", "num_cylinders", "engine_size", "fuel_system",
                   "bore", "stroke", "compression_ratio", "horsepower", "peak_rpm",
                   "city_mpg", "highway_mpg", "price"]
        df = retrieve_dataset('imports-85.csv', header=None, names=headers, na_values="?")
        df_y = df['price']
        df_X = df.drop(columns='price')
        df_train_X, df_test_X, df_train_y, df_test_y = train_test_split(df_X, df_y, test_size=0.2, random_state=7)
        # Encode strings to ordinal values
        categorical_col_names = list(df_train_X.select_dtypes(include='object').columns)
        categorical_col_indices = [df_train_X.columns.get_loc(col_name) for col_name in categorical_col_names]
        kwargs = {'num_leaves': 31, 'num_trees': 100, 'objective': 'regression',
                  'categorical_feature': categorical_col_indices}
        lgbm_regressor = LGBMRegressor(**kwargs)
        # Impute the x and y values
        imp_X = Imputer(missing_values=np.nan, strategy='most_frequent')
        imp_y = Imputer(missing_values=np.nan, strategy='mean')
        # reshape to 2D array since SimpleImputer can't work on 1D array
        df_train_y = df_train_y.values.reshape(df_train_y.shape[0], 1)
        imp_y.fit(df_train_y)
        imp_df_y = imp_y.transform(df_train_y)
        imp_X.fit(df_train_X)
        imp_train_X = pd.DataFrame(imp_X.transform(df_train_X))

        class CustomTextTransformer(BaseEstimator, TransformerMixin):
            def __init__(self):
                return

            def fit(self, X, y=None):
                return self

            def transform(self, X):
                return X.astype('U')

        custom_text = CustomTextTransformer()
        encoder = OneHotEncoder(handle_unknown='ignore', sparse=False)
        ct1 = ColumnTransformer([('cu', custom_text, categorical_col_indices)], remainder='passthrough')
        ct2 = ColumnTransformer([('ord', encoder, slice(0, len(categorical_col_indices)))], remainder='passthrough')
        pipeline = Pipeline([('cu', ct1), ('ct', ct2), ('lgbm', lgbm_regressor)])
        pipeline.fit(imp_train_X, imp_df_y[:, 0])
        if pass_categoricals:
            explainer = self.create_explainer(pipeline, imp_train_X, categorical_features=categorical_col_indices)
        else:
            explainer = self.create_explainer(pipeline, imp_train_X)
        explanation = explainer.explain_global(imp_X.transform(df_test_X))
        verify_serialization(explanation, exist_ok=True)
        # validate explanation when passing categoricals is for raw features for some specific explainers
        if pass_categoricals and verify_same_shape:
            assert np.array(explanation.local_importance_values).shape[1] == df_train_X.shape[1]
//  [python]
def verify_explain_model_categorical(self, pass_categoricals=False, verify_same_shape=False):
        headers = ["symboling", "normalized_losses", "make", "fuel_type", "aspiration",
                   "num_doors", "body_style", "drive_wheels", "engine_location",
                   "wheel_base", "length", "width", "height", "curb_weight",
                   "engine_type", "num_cylinders", "engine_size", "fuel_system",
                   "bore", "stroke", "compression_ratio", "horsepower", "peak_rpm",
                   "city_mpg", "highway_mpg", "price"]
        df = retrieve_dataset('imports-85.csv', header=None, names=headers, na_values="?")
        df_y = df['price']
        df_X = df.drop(columns='price')
        df_train_X, df_test_X, df_train_y, df_test_y = train_test_split(df_X, df_y, test_size=0.2, random_state=7)
        # Encode strings to ordinal values
        categorical_col_names = list(df_train_X.select_dtypes(include='object').columns)
        categorical_col_indices = [df_train_X.columns.get_loc(col_name) for col_name in categorical_col_names]
        kwargs = {'num_leaves': 31, 'num_trees': 100, 'objective': 'regression',
                  'categorical_feature': categorical_col_indices}
        lgbm_regressor = LGBMRegressor(**kwargs)
        # Impute the x and y values
        imp_X = SimpleImputer(missing_values=np.nan, strategy='most_frequent')
        imp_y = SimpleImputer(missing_values=np.nan, strategy='mean')
        # reshape to 2D array since SimpleImputer can't work on 1D array
        df_train_y = df_train_y.values.reshape(df_train_y.shape[0], 1)
        imp_y.fit(df_train_y)
        imp_df_y = imp_y.transform(df_train_y)
        imp_X.fit(df_train_X)
        imp_train_X = pd.DataFrame(imp_X.transform(df_train_X))

        class CustomTextTransformer(BaseEstimator, TransformerMixin):
            def __init__(self):
                return

            def fit(self, X, y=None):
                return self

            def transform(self, X):
                return X.astype('U')

        custom_text = CustomTextTransformer()
        encoder = OneHotEncoder(handle_unknown='ignore', sparse=False)
        ct1 = ColumnTransformer([('cu', custom_text, categorical_col_indices)], remainder='passthrough')
        ct2 = ColumnTransformer([('ord', encoder, slice(0, len(categorical_col_indices)))], remainder='passthrough')
        pipeline = Pipeline([('cu', ct1), ('ct', ct2), ('lgbm', lgbm_regressor)])
        pipeline.fit(imp_train_X, imp_df_y[:, 0])
        if pass_categoricals:
            explainer = self.create_explainer(pipeline, imp_train_X, categorical_features=categorical_col_indices)
        else:
            explainer = self.create_explainer(pipeline, imp_train_X)
        explanation = explainer.explain_global(imp_X.transform(df_test_X))
        verify_serialization(explanation, exist_ok=True)
        # validate explanation when passing categoricals is for raw features for some specific explainers
        if pass_categoricals and verify_same_shape:
            assert np.array(explanation.local_importance_values).shape[1] == df_train_X.shape[1]
//  [python]
def run():
    # Fetch data
    mnist = sklearn.datasets.fetch_mldata('MNIST original', data_home='./data')
    split = 60000
    X_train = np.reshape(mnist.data[:split], (-1, 1, 28, 28))/255.0
    y_train = mnist.target[:split]
    X_test = np.reshape(mnist.data[split:], (-1, 1, 28, 28))/255.0
    y_test = mnist.target[split:]
    n_classes = np.unique(y_train).size

    # Downsample training data
    n_train_samples = 3000
    train_idxs = np.random.random_integers(0, split-1, n_train_samples)
    X_train = X_train[train_idxs, ...]
    y_train = y_train[train_idxs, ...]

    # Setup convolutional neural network
    nn = nnet.NeuralNetwork(
        layers=[
            nnet.Conv(
                n_feats=12,
                filter_shape=(5, 5),
                strides=(1, 1),
                weight_scale=0.1,
                weight_decay=0.001,
            ),
            nnet.Activation('relu'),
            nnet.Pool(
                pool_shape=(2, 2),
                strides=(2, 2),
                mode='max',
            ),
            nnet.Conv(
                n_feats=16,
                filter_shape=(5, 5),
                strides=(1, 1),
                weight_scale=0.1,
                weight_decay=0.001,
            ),
            nnet.Activation('relu'),
            nnet.Flatten(),
            nnet.Linear(
                n_out=n_classes,
                weight_scale=0.1,
                weight_decay=0.02,
            ),
            nnet.LogRegression(),
        ],
    )

    # Train neural network
    t0 = time.time()
    nn.fit(X_train, y_train, learning_rate=0.05, max_iter=3, batch_size=32)
    t1 = time.time()
    print('Duration: %.1fs' % (t1-t0))

    # Evaluate on test data
    error = nn.error(X_test, y_test)
    print('Test error rate: %.4f' % error)
//  [python]
def run():
    # Fetch data
    mnist = sklearn.datasets.fetch_openml('mnist_784', version=1, data_home='./data')
    split = 60000
    X_train = np.reshape(mnist.data[:split], (-1, 1, 28, 28))/255.0
    y_train = mnist.target[:split]
    X_test = np.reshape(mnist.data[split:], (-1, 1, 28, 28))/255.0
    y_test = mnist.target[split:]
    n_classes = np.unique(y_train).size

    # Downsample training data
    n_train_samples = 3000
    train_idxs = np.random.random_integers(0, split-1, n_train_samples)
    X_train = X_train[train_idxs, ...]
    y_train = y_train[train_idxs, ...]

    # Setup convolutional neural network
    nn = nnet.NeuralNetwork(
        layers=[
            nnet.Conv(
                n_feats=12,
                filter_shape=(5, 5),
                strides=(1, 1),
                weight_scale=0.1,
                weight_decay=0.001,
            ),
            nnet.Activation('relu'),
            nnet.Pool(
                pool_shape=(2, 2),
                strides=(2, 2),
                mode='max',
            ),
            nnet.Conv(
                n_feats=16,
                filter_shape=(5, 5),
                strides=(1, 1),
                weight_scale=0.1,
                weight_decay=0.001,
            ),
            nnet.Activation('relu'),
            nnet.Flatten(),
            nnet.Linear(
                n_out=n_classes,
                weight_scale=0.1,
                weight_decay=0.02,
            ),
            nnet.LogRegression(),
        ],
    )

    # Train neural network
    t0 = time.time()
    nn.fit(X_train, y_train, learning_rate=0.05, max_iter=3, batch_size=32)
    t1 = time.time()
    print('Duration: %.1fs' % (t1-t0))

    # Evaluate on test data
    error = nn.error(X_test, y_test)
    print('Test error rate: %.4f' % error)
//  [python]
def mnist(digit='all', n_samples=0, return_gt=False):
    mnist = sk_datasets.fetch_mldata('MNIST original')
    X = mnist.data
    gt = mnist.target

    if digit == 'all':  # keep all digits
        pass
    else:
        X = X[gt == digit, :]
        gt = gt[gt == digit]

    if n_samples > len(X):
        raise ValueError('Requesting {} samples'
                         'from {} datapoints'.format(n_samples, len(X)))
    if n_samples > 0:
        np.random.seed(0)
        selection = np.random.randint(len(X), size=n_samples)
        X = X[selection, :]
        gt = gt[selection]

        idx = np.argsort(gt)
        X = X[idx, :]
        gt = gt[idx]

    if return_gt:
        return X, gt
    else:
        return X
//  [python]
def mnist(digit='all', n_samples=0, return_gt=False):
    mnist = sk_datasets.fetch_openml('mnist_784', version=1, as_frame=False)
    X = mnist.data
    gt = mnist.target

    if digit == 'all':  # keep all digits
        pass
    else:
        X = X[gt == str(digit), :]
        gt = gt[gt == str(digit)]

    if n_samples > len(X):
        raise ValueError('Requesting {} samples'
                         'from {} datapoints'.format(n_samples, len(X)))
    if n_samples > 0:
        np.random.seed(0)
        selection = np.random.randint(len(X), size=n_samples)
        X = X[selection, :]
        gt = gt[selection]

        idx = np.argsort(gt)
        X = X[idx, :]
        gt = gt[idx]

    if return_gt:
        return X, gt
    else:
        return X
//  [python]
def split_train_test(test_size):
    # choose a dataset with unbalanced class instances
    #data = sklearn.datasets.fetch_mldata('segment')
    data = sklearn.datasets.fetch_mldata('vehicle')

    X = StandardScaler().fit_transform(data['data'])
    target = np.unique(data['target'])
    # mapping the targets to 0 to n_classes-1
    y = np.array([np.where(target == i)[0][0] for i in data['target']])

    X_trn, X_tst, y_trn, y_tst = \
        train_test_split(X, y, test_size=test_size, stratify=y)

    # making sure each class appears ones initially
    init_y_ind = np.array(
        [np.where(y_trn == i)[0][0] for i in range(len(target))])
    y_ind = np.array([i for i in range(len(X_trn)) if i not in init_y_ind])
    trn_ds = Dataset(
        np.vstack((X_trn[init_y_ind], X_trn[y_ind])),
        np.concatenate((y_trn[init_y_ind], [None] * (len(y_ind)))))

    tst_ds = Dataset(X_tst, y_tst)

    fully_labeled_trn_ds = Dataset(
        np.vstack((X_trn[init_y_ind], X_trn[y_ind])),
        np.concatenate((y_trn[init_y_ind], y_trn[y_ind])))

    cost_matrix = 2000. * np.random.rand(len(target), len(target))
    np.fill_diagonal(cost_matrix, 0)

    return trn_ds, tst_ds, fully_labeled_trn_ds, cost_matrix
//  [python]
def split_train_test(test_size):
    # choose a dataset with unbalanced class instances
    #data = sklearn.datasets.fetch_mldata('segment')
    data = sklearn.datasets.fetch_openml(name='vehicle')

    X = StandardScaler().fit_transform(data['data'])
    target = np.unique(data['target'])
    # mapping the targets to 0 to n_classes-1
    y = np.array([np.where(target == i)[0][0] for i in data['target']])

    X_trn, X_tst, y_trn, y_tst = \
        train_test_split(X, y, test_size=test_size, stratify=y)

    # making sure each class appears ones initially
    init_y_ind = np.array(
        [np.where(y_trn == i)[0][0] for i in range(len(target))])
    y_ind = np.array([i for i in range(len(X_trn)) if i not in init_y_ind])
    trn_ds = Dataset(
        np.vstack((X_trn[init_y_ind], X_trn[y_ind])),
        np.concatenate((y_trn[init_y_ind], [None] * (len(y_ind)))))

    tst_ds = Dataset(X_tst, y_tst)

    fully_labeled_trn_ds = Dataset(
        np.vstack((X_trn[init_y_ind], X_trn[y_ind])),
        np.concatenate((y_trn[init_y_ind], y_trn[y_ind])))

    cost_matrix = 2000. * np.random.rand(len(target), len(target))
    np.fill_diagonal(cost_matrix, 0)

    return trn_ds, tst_ds, fully_labeled_trn_ds, cost_matrix
//  [python]
def example1():	
	# Other example: iris = fetch_mldata('iris', data_home=test_data_home)
	# test_data_home is directory for saving mat file
	mnist = datasets.fetch_mldata("MNIST Original")	
	x = mnist.data
	y = mnist.target
	print(mnist.keys()) # dict_keys(['COL_NAMES', 'data', 'target', 'DESCR'])
	
	x_train, x_test, y_train, y_test = train_test_split(x, y,
                                                        test_size=0.33,
                                                        random_state=42)
		
	_, D = x_train.shape	
	W = int(math.sqrt(D))	
	assert D == 784 and W == 28
		
	imageData = x_train.reshape((-1, W, W))	# picture 28 x 28
	plotExampleImg("Example: 1", imageData, y_train)
//  [python]
def example1():	
	# Other example: iris = fetch_mldata('iris', data_home=test_data_home)
	# test_data_home is directory for saving mat file
	mnist = datasets.fetch_openml("MNIST_784", version=1)	
	x = mnist.data
	y = mnist.target
	print(mnist.keys()) # dict_keys(['COL_NAMES', 'data', 'target', 'DESCR'])
	
	x_train, x_test, y_train, y_test = train_test_split(x, y,
                                                        test_size=0.33,
                                                        random_state=42)
		
	_, D = x_train.shape	
	W = int(math.sqrt(D))	
	assert D == 784 and W == 28
		
	imageData = x_train.reshape((-1, W, W))	# picture 28 x 28
	plotExampleImg("Example: 1", imageData, y_train)
//  [python]
def mnist(random_state=42):
    """
    x is in [0, 1] with shape (b, 1, 28, 28) and dtype floatX
    y is an int32 vector in range(10)
    """
    raw = sklearn.datasets.fetch_mldata('MNIST original')
    # rescaling to [0, 1] instead of [0, 255]
    x = raw['data'].reshape(-1, 1, 28, 28).astype(fX) / 255.0
    y = raw['target'].astype("int32")
    # NOTE: train data is initially in order of 0 through 9
    x1, x2, y1, y2 = sklearn.cross_validation.train_test_split(
        x[:60000],
        y[:60000],
        random_state=random_state,
        test_size=10000)
    train = {"x": x1, "y": y1}
    valid = {"x": x2, "y": y2}
    # NOTE: test data is in order of 0 through 9
    test = {"x": x[60000:], "y": y[60000:]}
    return train, valid, test
//  [python]
def mnist(random_state=42):
    """
    x is in [0, 1] with shape (b, 1, 28, 28) and dtype floatX
    y is an int32 vector in range(10)
    """
    raw = sklearn.datasets.fetch_openml('mnist_784', version=1)
    # rescaling to [0, 1] instead of [0, 255]
    x = raw['data'].reshape(-1, 1, 28, 28).astype(fX) / 255.0
    y = raw['target'].astype("int32")
    # NOTE: train data is initially in order of 0 through 9
    x1, x2, y1, y2 = sklearn.cross_validation.train_test_split(
        x[:60000],
        y[:60000],
        random_state=random_state,
        test_size=10000)
    train = {"x": x1, "y": y1}
    valid = {"x": x2, "y": y2}
    # NOTE: test data is in order of 0 through 9
    test = {"x": x[60000:], "y": y[60000:]}
    return train, valid, test
//  [python]
def _get_mnist_raw():
    def _download_from_other_source():
        # Attempt to download mnist data from another source
        url = 'https://dl.dropboxusercontent.com/s/b1kph20d20ybjk6/mnist-original.mat'
        warnings.warn('Could not download from mldata.org, attempting '
                      'to download from <%s>.' % url)
        file_name = os.path.join(_DOWNLOAD_DIR, 'mldata/mnist-original.mat')
        urllib.request.urlretrieve(url, file_name)

    _make_dir(_DOWNLOAD_DIR)
    n_attempts = 3
    print('Attempting to load/fetch MNIST data via sklearn.datasets.fetch_mldata')
    data_obj = None
    for i in range(n_attempts):
        try:
            data_obj = fetch_mldata('MNIST original', data_home=_DOWNLOAD_DIR)
        except (ConnectionResetError, urllib.error.HTTPError):
            _download_from_other_source()
            if i == n_attempts - 1:
                warnings.warn(
                    'Attempted to retrieve MNIST data from mldata.org or alternative %d times '
                    'and failed each time, please check internet connection' % n_attempts)
                raise
        else:
            # print('Successfully fetched MNIST data')
            break
    return data_obj.data, data_obj.target
//  [python]
def _get_mnist_raw():
    def _download_from_other_source():
        # Attempt to download mnist data from another source
        url = 'https://dl.dropboxusercontent.com/s/b1kph20d20ybjk6/mnist-original.mat'
        warnings.warn('Could not download from mldata.org, attempting '
                      'to download from <%s>.' % url)
        file_name = os.path.join(_DOWNLOAD_DIR, 'mldata/mnist-original.mat')
        urllib.request.urlretrieve(url, file_name)

    _make_dir(_DOWNLOAD_DIR)
    n_attempts = 3
    print('Attempting to load/fetch MNIST data via sklearn.datasets.fetch_openml')
    data_obj = None
    for i in range(n_attempts):
        try:
            data_obj = fetch_openml('mnist_784', version=1, data_home=_DOWNLOAD_DIR)
        except (ConnectionResetError, urllib.error.HTTPError):
            _download_from_other_source()
            if i == n_attempts - 1:
                warnings.warn(
                    'Attempted to retrieve MNIST data from mldata.org or alternative %d times '
                    'and failed each time, please check internet connection' % n_attempts)
                raise
        else:
            # print('Successfully fetched MNIST data')
            break
    return data_obj.data, data_obj.target
//  [python]
def test_fetch_openml_iris_pandas_equal_to_no_frame(monkeypatch):
    # as_frame = True returns the same underlying data as as_frame = False
    pytest.importorskip('pandas')
    data_id = 61

    _monkey_patch_webbased_functions(monkeypatch, data_id, True)

    frame_bunch = fetch_openml(data_id=data_id, as_frame=True, cache=False)
    frame_data = frame_bunch.data
    frame_target = frame_bunch.target

    norm_bunch = fetch_openml(data_id=data_id, as_frame=False, cache=False)
    norm_data = norm_bunch.data
    norm_target = norm_bunch.target

    assert_allclose(norm_data, frame_data)
    assert_array_equal(norm_target, frame_target)
//  [python]
def _test_features_list(data_id):
    # XXX Test is intended to verify/ensure correct decoding behavior
    # Not usable with sparse data or datasets that have columns marked as
    # {row_identifier, ignore}
    def decode_column(data_bunch, col_idx):
        col_name = data_bunch.feature_names[col_idx]
        if col_name in data_bunch.categories:
            # XXX: This would be faster with np.take, although it does not
            # handle missing values fast (also not with mode='wrap')
            cat = data_bunch.categories[col_name]
            result = [None if is_scalar_nan(idx) else cat[int(idx)]
                      for idx in data_bunch.data[:, col_idx]]
            return np.array(result, dtype='O')
        else:
            # non-nominal attribute
            return data_bunch.data[:, col_idx]

    data_bunch = fetch_openml(data_id=data_id, cache=False,
                              target_column=None, as_frame=False)

    # also obtain decoded arff
    data_description = _get_data_description_by_id(data_id, None)
    sparse = data_description['format'].lower() == 'sparse_arff'
    if sparse is True:
        raise ValueError('This test is not intended for sparse data, to keep '
                         'code relatively simple')
    url = _DATA_FILE.format(data_description['file_id'])
    with _open_openml_url(url, data_home=None) as f:
        data_arff = _arff.load((line.decode('utf-8') for line in f),
                               return_type=(_arff.COO if sparse
                                            else _arff.DENSE_GEN),
                               encode_nominal=False)

    data_downloaded = np.array(list(data_arff['data']), dtype='O')

    for i in range(len(data_bunch.feature_names)):
        # XXX: Test per column, as this makes it easier to avoid problems with
        # missing values

        np.testing.assert_array_equal(data_downloaded[:, i],
                                      decode_column(data_bunch, i))
//  [python]
def test_fetch_openml_iris_pandas(monkeypatch):
    # classification dataset with numeric only columns
    pd = pytest.importorskip('pandas')
    CategoricalDtype = pd.api.types.CategoricalDtype
    data_id = 61
    data_shape = (150, 4)
    target_shape = (150, )
    frame_shape = (150, 5)

    target_dtype = CategoricalDtype(['Iris-setosa', 'Iris-versicolor',
                                     'Iris-virginica'])
    data_dtypes = [np.float64] * 4
    data_names = ['sepallength', 'sepalwidth', 'petallength', 'petalwidth']
    target_name = 'class'

    _monkey_patch_webbased_functions(monkeypatch, data_id, True)

    bunch = fetch_openml(data_id=data_id, as_frame=True, cache=False)
    data = bunch.data
    target = bunch.target
    frame = bunch.frame

    assert isinstance(data, pd.DataFrame)
    assert np.all(data.dtypes == data_dtypes)
    assert data.shape == data_shape
    assert np.all(data.columns == data_names)
    assert np.all(bunch.feature_names == data_names)
    assert bunch.target_names == [target_name]

    assert isinstance(target, pd.Series)
    assert target.dtype == target_dtype
    assert target.shape == target_shape
    assert target.name == target_name
    assert target.index.is_unique

    assert isinstance(frame, pd.DataFrame)
    assert frame.shape == frame_shape
    assert np.all(frame.dtypes == data_dtypes + [target_dtype])
    assert frame.index.is_unique
//  [python]
def test_fetch_openml_anneal_pandas(monkeypatch):
    # classification dataset with numeric and categorical columns
    pd = pytest.importorskip('pandas')
    CategoricalDtype = pd.api.types.CategoricalDtype

    data_id = 2
    target_column = 'class'
    data_shape = (11, 38)
    target_shape = (11,)
    frame_shape = (11, 39)
    expected_data_categories = 32
    expected_data_floats = 6

    _monkey_patch_webbased_functions(monkeypatch, data_id, True)

    bunch = fetch_openml(data_id=data_id, as_frame=True,
                         target_column=target_column, cache=False)
    data = bunch.data
    target = bunch.target
    frame = bunch.frame

    assert isinstance(data, pd.DataFrame)
    assert data.shape == data_shape
    n_categories = len([dtype for dtype in data.dtypes
                       if isinstance(dtype, CategoricalDtype)])
    n_floats = len([dtype for dtype in data.dtypes if dtype.kind == 'f'])
    assert expected_data_categories == n_categories
    assert expected_data_floats == n_floats

    assert isinstance(target, pd.Series)
    assert target.shape == target_shape
    assert isinstance(target.dtype, CategoricalDtype)

    assert isinstance(frame, pd.DataFrame)
    assert frame.shape == frame_shape
//  [python]
def test_fetch_openml_iris_multitarget_pandas(monkeypatch):
    # classification dataset with numeric only columns
    pd = pytest.importorskip('pandas')
    CategoricalDtype = pd.api.types.CategoricalDtype
    data_id = 61
    data_shape = (150, 3)
    target_shape = (150, 2)
    frame_shape = (150, 5)
    target_column = ['petalwidth', 'petallength']

    cat_dtype = CategoricalDtype(['Iris-setosa', 'Iris-versicolor',
                                  'Iris-virginica'])
    data_dtypes = [np.float64, np.float64] + [cat_dtype]
    data_names = ['sepallength', 'sepalwidth', 'class']
    target_dtypes = [np.float64, np.float64]
    target_names = ['petalwidth', 'petallength']

    _monkey_patch_webbased_functions(monkeypatch, data_id, True)

    bunch = fetch_openml(data_id=data_id, as_frame=True, cache=False,
                         target_column=target_column)
    data = bunch.data
    target = bunch.target
    frame = bunch.frame

    assert isinstance(data, pd.DataFrame)
    assert np.all(data.dtypes == data_dtypes)
    assert data.shape == data_shape
    assert np.all(data.columns == data_names)
    assert np.all(bunch.feature_names == data_names)
    assert bunch.target_names == target_names

    assert isinstance(target, pd.DataFrame)
    assert np.all(target.dtypes == target_dtypes)
    assert target.shape == target_shape
    assert np.all(target.columns == target_names)

    assert isinstance(frame, pd.DataFrame)
    assert frame.shape == frame_shape
    assert np.all(frame.dtypes == [np.float64] * 4 + [cat_dtype])
//  [python]
def test_fetch_openml_australian_pandas_error_sparse(monkeypatch):
    data_id = 292

    _monkey_patch_webbased_functions(monkeypatch, data_id, True)

    msg = 'Cannot return dataframe with sparse data'
    with pytest.raises(ValueError, match=msg):
        fetch_openml(data_id=data_id, as_frame=True, cache=False)
//  [python]
def _save_mnist_recreation_indices():
    """Code to find MNIST train, validation and test indices for recreation of
    MNIST MAF dataset.
    Note this should not be called directly.  This is only here for reproducibility."""
    warnings.warn('This function should generally not be called because it '
                  'requires special setup but is kept here in order to reproduce functions if '
                  'needed.')
    # Import maf data
    datasets_root = os.path.join(
        os.path.dirname(os.path.realpath(__file__)),
        '..', '..', 'maf', 'data',
    )
    mnist_path = os.path.join(datasets_root, 'mnist', 'mnist.pkl.gz')
    with gzip.open(mnist_path, 'rb') as f:
        maf_train, maf_val, maf_test = pickle.load(f)

    # Import raw mnist data
    data_obj = fetch_mldata('MNIST original')  # , data_home=custom_data_home)

    # Prepare comparison matrices
    X_all = data_obj.data / 256.0
    y_all = data_obj.target

    maf_data_tuple = (maf_train[0], maf_val[0], maf_test[0])
    n_maf = [X.shape[0] for X in maf_data_tuple]
    X_maf = np.vstack(maf_data_tuple)
    y_maf = np.concatenate((maf_train[1], maf_val[1], maf_test[1]))

    # Sort maf using all columns
    mnist_ind = np.lexsort(np.hstack((X_all, y_all.reshape(-1, 1))).T)
    maf_ind = np.lexsort(np.hstack((X_maf, y_maf.reshape(-1, 1))).T)
    rev_maf_ind = np.argsort(maf_ind)

    # Show that matrices match when sorted by indices
    print('Checking if the datasets are the same (should all be 0)')

    def n_diff(X, Y):
        """

        Parameters
        ----------
        X :
        Y :

        Returns
        -------

        """
        return np.count_nonzero(X - Y)

    def print_n_diff(X, Y):
        """

        Parameters
        ----------
        X :
        Y :
        """
        print('Number different = %d' % n_diff(X, Y))

    print_n_diff(X_all[mnist_ind], X_maf[maf_ind])
    print_n_diff(y_all[mnist_ind], y_maf[maf_ind])

    # Retrieve indices and show that they are the same
    train_idx, val_idx, test_idx = (
        mnist_ind[
            rev_maf_ind[np.sum(n_maf[:i], dtype=np.int):np.sum(n_maf[:(i + 1)], dtype=np.int)]]
        for i in range(3)
    )
    for idx, maf in zip((train_idx, val_idx, test_idx), (maf_train, maf_val, maf_test)):
        print_n_diff(X_all[idx], maf[0])
        print_n_diff(y_all[idx], maf[1])

    gzip_file = os.path.join(
        os.path.dirname(os.path.realpath(__file__)),
        'maf_mnist_splits.txt.gz'
    )
    with gzip.open(gzip_file, 'w+') as f:
        f.write('# Indices of MNIST dataset retrieved using '
                'sklearn.datasets.fetch_mldata(\'MNIST original\') that correspond to the train, '
                'validation and test sets of the MAF paper (one line each).\n')
        for i, idx in enumerate([train_idx, val_idx, test_idx]):
            s = str(idx.tolist())
            s = s[1:-1]  # Trim off ends
            f.write(s)
            if i < 2:
                f.write('\n')
//  [python]
def _save_mnist_recreation_indices():
    """Code to find MNIST train, validation and test indices for recreation of
    MNIST MAF dataset.
    Note this should not be called directly.  This is only here for reproducibility."""
    warnings.warn('This function should generally not be called because it '
                  'requires special setup but is kept here in order to reproduce functions if '
                  'needed.')
    # Import maf data
    datasets_root = os.path.join(
        os.path.dirname(os.path.realpath(__file__)),
        '..', '..', 'maf', 'data',
    )
    mnist_path = os.path.join(datasets_root, 'mnist', 'mnist.pkl.gz')
    with gzip.open(mnist_path, 'rb') as f:
        maf_train, maf_val, maf_test = pickle.load(f)

    # Import raw mnist data
    data_obj = fetch_openml('mnist_784', version=1)  # , data_home=custom_data_home)

    # Prepare comparison matrices
    X_all = data_obj.data / 256.0
    y_all = data_obj.target

    maf_data_tuple = (maf_train[0], maf_val[0], maf_test[0])
    n_maf = [X.shape[0] for X in maf_data_tuple]
    X_maf = np.vstack(maf_data_tuple)
    y_maf = np.concatenate((maf_train[1], maf_val[1], maf_test[1]))

    # Sort maf using all columns
    mnist_ind = np.lexsort(np.hstack((X_all, y_all.reshape(-1, 1))).T)
    maf_ind = np.lexsort(np.hstack((X_maf, y_maf.reshape(-1, 1))).T)
    rev_maf_ind = np.argsort(maf_ind)

    # Show that matrices match when sorted by indices
    print('Checking if the datasets are the same (should all be 0)')

    def n_diff(X, Y):
        """

        Parameters
        ----------
        X :
        Y :

        Returns
        -------

        """
        return np.count_nonzero(X - Y)

    def print_n_diff(X, Y):
        """

        Parameters
        ----------
        X :
        Y :
        """
        print('Number different = %d' % n_diff(X, Y))

    print_n_diff(X_all[mnist_ind], X_maf[maf_ind])
    print_n_diff(y_all[mnist_ind], y_maf[maf_ind])

    # Retrieve indices and show that they are the same
    train_idx, val_idx, test_idx = (
        mnist_ind[
            rev_maf_ind[np.sum(n_maf[:i], dtype=np.int):np.sum(n_maf[:(i + 1)], dtype=np.int)]]
        for i in range(3)
    )
    for idx, maf in zip((train_idx, val_idx, test_idx), (maf_train, maf_val, maf_test)):
        print_n_diff(X_all[idx], maf[0])
        print_n_diff(y_all[idx], maf[1])

    gzip_file = os.path.join(
        os.path.dirname(os.path.realpath(__file__)),
        'maf_mnist_splits.txt.gz'
    )
    with gzip.open(gzip_file, 'w+') as f:
        f.write('# Indices of MNIST dataset retrieved using '
                'sklearn.datasets.fetch_openml(\'mnist_784\', version=1) that correspond to the train, '
                'validation and test sets of the MAF paper (one line each).\n')
        for i, idx in enumerate([train_idx, val_idx, test_idx]):
            s = str(idx.tolist())
            s = s[1:-1]  # Trim off ends
            f.write(s)
            if i < 2:
                f.write('\n')
//  [python]
def test_fetch_openml_as_frame_auto(monkeypatch):
    pd = pytest.importorskip('pandas')

    data_id = 61  # iris dataset version 1
    _monkey_patch_webbased_functions(monkeypatch, data_id, True)
    data = fetch_openml(data_id=data_id, as_frame='auto')
    assert isinstance(data.data, pd.DataFrame)

    data_id = 292  # Australian dataset version 1
    _monkey_patch_webbased_functions(monkeypatch, data_id, True)
    data = fetch_openml(data_id=data_id, as_frame='auto')
    assert isinstance(data.data, scipy.sparse.csr_matrix)
//  [python]
def test_convert_arff_data_dataframe_warning_low_memory_pandas(monkeypatch):
    pytest.importorskip('pandas')

    data_id = 1119
    _monkey_patch_webbased_functions(monkeypatch, data_id, True)

    msg = 'Could not adhere to working_memory config.'
    with pytest.warns(UserWarning, match=msg):
        with config_context(working_memory=1e-6):
            fetch_openml(data_id=data_id, as_frame=True, cache=False)
//  [python]
def test_fetch_openml_cpu_pandas(monkeypatch):
    # regression dataset with numeric and categorical columns
    pd = pytest.importorskip('pandas')
    CategoricalDtype = pd.api.types.CategoricalDtype
    data_id = 561
    data_shape = (209, 7)
    target_shape = (209, )
    frame_shape = (209, 8)

    cat_dtype = CategoricalDtype(['adviser', 'amdahl', 'apollo', 'basf',
                                  'bti', 'burroughs', 'c.r.d', 'cdc',
                                  'cambex', 'dec', 'dg', 'formation',
                                  'four-phase', 'gould', 'hp', 'harris',
                                  'honeywell', 'ibm', 'ipl', 'magnuson',
                                  'microdata', 'nas', 'ncr', 'nixdorf',
                                  'perkin-elmer', 'prime', 'siemens',
                                  'sperry', 'sratus', 'wang'])
    data_dtypes = [cat_dtype] + [np.float64] * 6
    feature_names = ['vendor', 'MYCT', 'MMIN', 'MMAX', 'CACH',
                     'CHMIN', 'CHMAX']
    target_name = 'class'

    _monkey_patch_webbased_functions(monkeypatch, data_id, True)
    bunch = fetch_openml(data_id=data_id, as_frame=True, cache=False)
    data = bunch.data
    target = bunch.target
    frame = bunch.frame

    assert isinstance(data, pd.DataFrame)
    assert data.shape == data_shape
    assert np.all(data.dtypes == data_dtypes)
    assert np.all(data.columns == feature_names)
    assert np.all(bunch.feature_names == feature_names)
    assert bunch.target_names == [target_name]

    assert isinstance(target, pd.Series)
    assert target.shape == target_shape
    assert target.dtype == np.float64
    assert target.name == target_name

    assert isinstance(frame, pd.DataFrame)
    assert frame.shape == frame_shape
//  [python]
def test_fetch_openml_adultcensus_pandas_return_X_y(monkeypatch):
    pd = pytest.importorskip('pandas')
    CategoricalDtype = pd.api.types.CategoricalDtype

    data_id = 1119
    data_shape = (10, 14)
    target_shape = (10, )

    expected_data_categories = 8
    expected_data_floats = 6
    target_column = 'class'

    _monkey_patch_webbased_functions(monkeypatch, data_id, True)
    X, y = fetch_openml(data_id=data_id, as_frame=True, cache=False,
                        return_X_y=True)
    assert isinstance(X, pd.DataFrame)
    assert X.shape == data_shape
    n_categories = len([dtype for dtype in X.dtypes
                       if isinstance(dtype, CategoricalDtype)])
    n_floats = len([dtype for dtype in X.dtypes if dtype.kind == 'f'])
    assert expected_data_categories == n_categories
    assert expected_data_floats == n_floats

    assert isinstance(y, pd.Series)
    assert y.shape == target_shape
    assert y.name == target_column
//  [python]
def test_fetch_openml_adultcensus_pandas(monkeypatch):
    pd = pytest.importorskip('pandas')
    CategoricalDtype = pd.api.types.CategoricalDtype

    # Check because of the numeric row attribute (issue #12329)
    data_id = 1119
    data_shape = (10, 14)
    target_shape = (10, )
    frame_shape = (10, 15)

    expected_data_categories = 8
    expected_data_floats = 6
    target_column = 'class'

    _monkey_patch_webbased_functions(monkeypatch, data_id, True)
    bunch = fetch_openml(data_id=data_id, as_frame=True, cache=False)
    data = bunch.data
    target = bunch.target
    frame = bunch.frame

    assert isinstance(data, pd.DataFrame)
    assert data.shape == data_shape
    n_categories = len([dtype for dtype in data.dtypes
                       if isinstance(dtype, CategoricalDtype)])
    n_floats = len([dtype for dtype in data.dtypes if dtype.kind == 'f'])
    assert expected_data_categories == n_categories
    assert expected_data_floats == n_floats

    assert isinstance(target, pd.Series)
    assert target.shape == target_shape
    assert target.name == target_column

    assert isinstance(frame, pd.DataFrame)
    assert frame.shape == frame_shape
//  [python]
def test_fetch_openml_cache(monkeypatch, gzip_response, tmpdir):
    def _mock_urlopen_raise(request):
        raise ValueError('This mechanism intends to test correct cache'
                         'handling. As such, urlopen should never be '
                         'accessed. URL: %s' % request.get_full_url())
    data_id = 2
    cache_directory = str(tmpdir.mkdir('scikit_learn_data'))
    _monkey_patch_webbased_functions(
        monkeypatch, data_id, gzip_response)
    X_fetched, y_fetched = fetch_openml(data_id=data_id, cache=True,
                                        data_home=cache_directory,
                                        return_X_y=True, as_frame=False)

    monkeypatch.setattr(sklearn.datasets._openml, 'urlopen',
                        _mock_urlopen_raise)

    X_cached, y_cached = fetch_openml(data_id=data_id, cache=True,
                                      data_home=cache_directory,
                                      return_X_y=True, as_frame=False)
    np.testing.assert_array_equal(X_fetched, X_cached)
    np.testing.assert_array_equal(y_fetched, y_cached)
//  [python]
def test_fetch_openml_miceprotein_pandas(monkeypatch):
    # JvR: very important check, as this dataset defined several row ids
    # and ignore attributes. Note that data_features json has 82 attributes,
    # and row id (1), ignore attributes (3) have been removed.
    pd = pytest.importorskip('pandas')
    CategoricalDtype = pd.api.types.CategoricalDtype

    data_id = 40966
    data_shape = (7, 77)
    target_shape = (7, )
    frame_shape = (7, 78)

    target_column = 'class'
    frame_n_categories = 1
    frame_n_floats = 77

    _monkey_patch_webbased_functions(monkeypatch, data_id, True)
    bunch = fetch_openml(data_id=data_id, as_frame=True, cache=False)
    data = bunch.data
    target = bunch.target
    frame = bunch.frame

    assert isinstance(data, pd.DataFrame)
    assert data.shape == data_shape
    assert np.all(data.dtypes == np.float64)

    assert isinstance(target, pd.Series)
    assert isinstance(target.dtype, CategoricalDtype)
    assert target.shape == target_shape
    assert target.name == target_column

    assert isinstance(frame, pd.DataFrame)
    assert frame.shape == frame_shape
    n_categories = len([dtype for dtype in frame.dtypes
                       if isinstance(dtype, CategoricalDtype)])
    n_floats = len([dtype for dtype in frame.dtypes if dtype.kind == 'f'])
    assert frame_n_categories == n_categories
    assert frame_n_floats == n_floats