Repository: openml/openml-python

Issue:
pickle for SVHN bigger than arff file
For dataset 41081, I get:
```
1.1G	dataset.arff
2.3G	dataset.pkl.py3
```
Maybe we should be using joblib instead? My hard drive is getting full because I'm trying to run CC-18 stuff, which is a bit annoying.

Discussion:
PGijsbers: To be clear; this PR supersedes #892 (I imagine because it's easier than dealing with merge conflicts)?
ArlindKadra: > To be clear; this PR supersedes #892 (I imagine because it's easier than dealing with merge conflicts)?

@PGijsbers yes exactly, the last commit was nearly 1 year ago.

Patch:
diff --git a/doc/progress.rst b/doc/progress.rst
index 7dc63334..2e077484 100644
--- a/doc/progress.rst
+++ b/doc/progress.rst
@@ -8,6 +8,7 @@ Changelog
 
 0.11.1
 ~~~~~~
+* MAINT #891: Changed the way that numerical features are stored. Numerical features that range from 0 to 255 are now stored as uint8, which reduces the storage space required as well as storing and loading times.
 * MAINT #671: Improved the performance of ``check_datasets_active`` by only querying the given list of datasets in contrast to querying all datasets. Modified the corresponding unit test.
 
 0.11.0
diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py
index 8c366dfb..a51603d8 100644
--- a/openml/datasets/dataset.py
+++ b/openml/datasets/dataset.py
@@ -407,7 +407,7 @@ class OpenMLDataset(OpenMLBase):
         categories_names = {}
         categorical = []
         for i, (name, type_) in enumerate(data["attributes"]):
-            # if the feature is nominal and the a sparse matrix is
+            # if the feature is nominal and a sparse matrix is
             # requested, the categories need to be numeric
             if isinstance(type_, list) and self.format.lower() == "sparse_arff":
                 try:
@@ -456,6 +456,18 @@ class OpenMLDataset(OpenMLBase):
                     col.append(
                         self._unpack_categories(X[column_name], categories_names[column_name])
                     )
+                elif attribute_dtype[column_name] in ('floating',
+                                                      'integer'):
+                    X_col = X[column_name]
+                    if X_col.min() >= 0 and X_col.max() <= 255:
+                        try:
+                            X_col_uint = X_col.astype('uint8')
+                            if (X_col == X_col_uint).all():
+                                col.append(X_col_uint)
+                                continue
+                        except ValueError:
+                            pass
+                    col.append(X[column_name])
                 else:
                     col.append(X[column_name])
             X = pd.concat(col, axis=1)


Test patch:
diff --git a/tests/test_datasets/test_dataset.py b/tests/test_datasets/test_dataset.py
index 73dbfa13..82a90154 100644
--- a/tests/test_datasets/test_dataset.py
+++ b/tests/test_datasets/test_dataset.py
@@ -72,13 +72,13 @@ class OpenMLDatasetTest(TestBase):
         self.assertEqual(data.shape[1], len(self.titanic.features))
         self.assertEqual(data.shape[0], 1309)
         col_dtype = {
-            "pclass": "float64",
+            "pclass": "uint8",
             "survived": "category",
             "name": "object",
             "sex": "category",
             "age": "float64",
-            "sibsp": "float64",
-            "parch": "float64",
+            "sibsp": "uint8",
+            "parch": "uint8",
             "ticket": "object",
             "fare": "float64",
             "cabin": "object",
@@ -118,21 +118,29 @@ class OpenMLDatasetTest(TestBase):
         with pytest.raises(PyOpenMLError, match=err_msg):
             self.titanic.get_data(dataset_format="array")
 
+    def _check_expected_type(self, dtype, is_cat, col):
+        if is_cat:
+            expected_type = 'category'
+        elif not col.isna().any() and (col.astype('uint8') == col).all():
+            expected_type = 'uint8'
+        else:
+            expected_type = 'float64'
+
+        self.assertEqual(dtype.name, expected_type)
+
     def test_get_data_with_rowid(self):
         self.dataset.row_id_attribute = "condition"
         rval, _, categorical, _ = self.dataset.get_data(include_row_id=True)
         self.assertIsInstance(rval, pd.DataFrame)
-        for (dtype, is_cat) in zip(rval.dtypes, categorical):
-            expected_type = "category" if is_cat else "float64"
-            self.assertEqual(dtype.name, expected_type)
+        for (dtype, is_cat, col) in zip(rval.dtypes, categorical, rval):
+            self._check_expected_type(dtype, is_cat, rval[col])
         self.assertEqual(rval.shape, (898, 39))
         self.assertEqual(len(categorical), 39)
 
         rval, _, categorical, _ = self.dataset.get_data()
         self.assertIsInstance(rval, pd.DataFrame)
-        for (dtype, is_cat) in zip(rval.dtypes, categorical):
-            expected_type = "category" if is_cat else "float64"
-            self.assertEqual(dtype.name, expected_type)
+        for (dtype, is_cat, col) in zip(rval.dtypes, categorical, rval):
+            self._check_expected_type(dtype, is_cat, rval[col])
         self.assertEqual(rval.shape, (898, 38))
         self.assertEqual(len(categorical), 38)
 
@@ -149,9 +157,8 @@ class OpenMLDatasetTest(TestBase):
     def test_get_data_with_target_pandas(self):
         X, y, categorical, attribute_names = self.dataset.get_data(target="class")
         self.assertIsInstance(X, pd.DataFrame)
-        for (dtype, is_cat) in zip(X.dtypes, categorical):
-            expected_type = "category" if is_cat else "float64"
-            self.assertEqual(dtype.name, expected_type)
+        for (dtype, is_cat, col) in zip(X.dtypes, categorical, X):
+            self._check_expected_type(dtype, is_cat, X[col])
         self.assertIsInstance(y, pd.Series)
         self.assertEqual(y.dtype.name, "category")
 
@@ -174,16 +181,14 @@ class OpenMLDatasetTest(TestBase):
     def test_get_data_with_ignore_attributes(self):
         self.dataset.ignore_attribute = ["condition"]
         rval, _, categorical, _ = self.dataset.get_data(include_ignore_attribute=True)
-        for (dtype, is_cat) in zip(rval.dtypes, categorical):
-            expected_type = "category" if is_cat else "float64"
-            self.assertEqual(dtype.name, expected_type)
+        for (dtype, is_cat, col) in zip(rval.dtypes, categorical, rval):
+            self._check_expected_type(dtype, is_cat, rval[col])
         self.assertEqual(rval.shape, (898, 39))
         self.assertEqual(len(categorical), 39)
 
         rval, _, categorical, _ = self.dataset.get_data(include_ignore_attribute=False)
-        for (dtype, is_cat) in zip(rval.dtypes, categorical):
-            expected_type = "category" if is_cat else "float64"
-            self.assertEqual(dtype.name, expected_type)
+        for (dtype, is_cat, col) in zip(rval.dtypes, categorical, rval):
+            self._check_expected_type(dtype, is_cat, rval[col])
         self.assertEqual(rval.shape, (898, 38))
         self.assertEqual(len(categorical), 38)
 
diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py
index 38b035fc..87ec699c 100644
--- a/tests/test_datasets/test_dataset_functions.py
+++ b/tests/test_datasets/test_dataset_functions.py
@@ -373,6 +373,13 @@ class TestOpenMLDataset(TestBase):
         openml.config.server = self.production_server
         self.assertRaises(OpenMLPrivateDatasetError, openml.datasets.get_dataset, 45)
 
+    def test_get_dataset_uint8_dtype(self):
+        dataset = openml.datasets.get_dataset(1)
+        self.assertEqual(type(dataset), OpenMLDataset)
+        self.assertEqual(dataset.name, 'anneal')
+        df, _, _, _ = dataset.get_data()
+        self.assertEqual(df['carbon'].dtype, 'uint8')
+
     def test_get_dataset(self):
         # This is the only non-lazy load to ensure default behaviour works.
         dataset = openml.datasets.get_dataset(1)