Repository: chainer/chainer

Issue:
Stop using scikit-learn to just download MNIST
Current MNIST example uses scikit-learn to download MNIST. It is overkill and adds an installation overhead to users and example testing.



Discussion:

Resolving patch:
diff --git a/examples/mnist/download_convert.py b/examples/mnist/download_convert.py
new file mode 100755
--- /dev/null
+++ b/examples/mnist/download_convert.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+
+import gzip
+
+import numpy as np
+import six
+from six.moves.urllib import request
+
+parent = 'http://yann.lecun.com/exdb/mnist'
+train_images = 'train-images-idx3-ubyte.gz'
+train_labels = 'train-labels-idx1-ubyte.gz'
+test_images = 't10k-images-idx3-ubyte.gz'
+test_labels = 't10k-labels-idx1-ubyte.gz'
+num_train = 60000
+num_test = 10000
+dim = 784
+
+
+def load_mnist(images, labels, num):
+    data = np.zeros(num * dim, dtype=np.uint8).reshape((num, dim))
+    target = np.zeros(num, dtype=np.uint8).reshape((num, ))
+
+    with gzip.open(images, 'rb') as f_images,\
+            gzip.open(labels, 'rb') as f_labels:
+        f_images.read(16)
+        f_labels.read(8)
+        for i in six.moves.range(num):
+            target[i] = ord(f_labels.read(1))
+            for j in six.moves.range(dim):
+                data[i, j] = ord(f_images.read(1))
+
+    return data, target
+
+print('Downloading {:s}...'.format(train_images))
+request.urlretrieve('{:s}/{:s}'.format(parent, train_images), train_images)
+print('Done')
+print('Downloading {:s}...'.format(train_labels))
+request.urlretrieve('{:s}/{:s}'.format(parent, train_labels), train_labels)
+print('Done')
+print('Downloading {:s}...'.format(test_images))
+request.urlretrieve('{:s}/{:s}'.format(parent, test_images), test_images)
+print('Done')
+print('Downloading {:s}...'.format(test_labels))
+request.urlretrieve('{:s}/{:s}'.format(parent, test_labels), test_labels)
+print('Done')
+
+print('Converting training data...')
+data_train, target_train = load_mnist(train_images, train_labels, num_train)
+print('Done')
+print('Converting test data...')
+data_test, target_test = load_mnist(test_images, test_labels, num_test)
+mnist = {}
+mnist['data'] = np.append(data_train, data_test, axis=0)
+mnist['target'] = np.append(target_train, target_test, axis=0)
+
+print('Done')
+print('Save output...')
+with open('mnist.pkl', 'wb') as output:
+    six.moves.cPickle.dump(mnist, output, -1)
+print('Done')
+print('Convert completed')
diff --git a/examples/mnist/train_mnist.py b/examples/mnist/train_mnist.py
--- a/examples/mnist/train_mnist.py
+++ b/examples/mnist/train_mnist.py
@@ -9,7 +9,6 @@
 
 import numpy as np
 import six
-from sklearn.datasets import fetch_mldata
 
 import chainer
 from chainer import cuda
@@ -26,15 +25,16 @@
 n_units = 1000
 
 # Prepare dataset
-print('fetch MNIST dataset')
-mnist = fetch_mldata('MNIST original')
-mnist.data = mnist.data.astype(np.float32)
-mnist.data /= 255
-mnist.target = mnist.target.astype(np.int32)
+print('load MNIST dataset')
+with open('mnist.pkl', 'rb') as mnist_pickle:
+    mnist = six.moves.cPickle.load(mnist_pickle)
+mnist['data'] = mnist['data'].astype(np.float32)
+mnist['data'] /= 255
+mnist['target'] = mnist['target'].astype(np.int32)
 
 N = 60000
-x_train, x_test = np.split(mnist.data,   [N])
-y_train, y_test = np.split(mnist.target, [N])
+x_train, x_test = np.split(mnist['data'],   [N])
+y_train, y_test = np.split(mnist['target'], [N])
 N_test = y_test.size
 
 # Prepare multi-layer perceptron model
diff --git a/examples/mnist/train_mnist_model_parallel.py b/examples/mnist/train_mnist_model_parallel.py
--- a/examples/mnist/train_mnist_model_parallel.py
+++ b/examples/mnist/train_mnist_model_parallel.py
@@ -10,7 +10,6 @@
 
 import numpy as np
 import six
-from sklearn import datasets
 
 import chainer
 from chainer import cuda
@@ -23,15 +22,16 @@
 n_units = 2000
 
 # Prepare dataset
-print('fetch MNIST dataset')
-mnist = datasets.fetch_mldata('MNIST original')
-mnist.data = mnist.data.astype(np.float32)
-mnist.data /= 255
-mnist.target = mnist.target.astype(np.int32)
+print('load MNIST dataset')
+with open('mnist.pkl', 'rb') as mnist_pickle:
+    mnist = six.moves.cPickle.load(mnist_pickle)
+mnist['data'] = mnist['data'].astype(np.float32)
+mnist['data'] /= 255
+mnist['target'] = mnist['target'].astype(np.int32)
 
 N = 60000
-x_train, x_test = np.split(mnist.data,   [N])
-y_train, y_test = np.split(mnist.target, [N])
+x_train, x_test = np.split(mnist['data'],   [N])
+y_train, y_test = np.split(mnist['target'], [N])
 N_test = y_test.size
 
 # Prepare the multi-layer perceptron model


Test patch: