Repository: keras-team/autokeras

Issue:
Dataloader load from disk by batches
<!---
**If you are reporting a bug:**
*  Verify that your issue is not being currently addressed by other issues or pull requests.
*  Please note that Auto-Keras is only compatible with **Python 3.6**.
*  Tag the issue with the `bug report` tag.
-->

### Bug Description

After the first model training, in the second model, I am getting the Memory error. I have also added the latest release as in another issue it was mentioned that memory error bug has been fixed in the latest issue. The error looks as follows: 
Traceback (most recent call last):                                         
  File "/home/monika/.local/lib/python3.6/site-packages/tqdm/_tqdm.py", line 885, in __del__
    self.close()
  File "/home/monika/.local/lib/python3.6/site-packages/tqdm/_tqdm.py", line 1090, in close
    self._decr_instances(self)
  File "/home/monika/.local/lib/python3.6/site-packages/tqdm/_tqdm.py", line 454, in _decr_instances
    cls.monitor.exit()
  File "/home/monika/.local/lib/python3.6/site-packages/tqdm/_monitor.py", line 52, in exit
    self.join()
  File "/usr/lib/python3.6/threading.py", line 1053, in join
    raise RuntimeError("cannot join current thread")
RuntimeError: cannot join current thread
/usr/lib/python3.6/multiprocessing/semaphore_tracker.py:143: UserWarning: semaphore_tracker: There appear to be 1 leaked semaphores to clean up at shutdown
  len(cache))
Traceback (most recent call last):
  File "mnist.py", line 21, in <module>
    clf.fit(x_train, y_train, time_limit=12 * 60 * 60)
  File "/home/monika/.local/lib/python3.6/site-packages/autokeras/image_supervised.py", line 239, in fit
    run_searcher_once(train_data, test_data, self.path, int(time_remain))
  File "/home/monika/.local/lib/python3.6/site-packages/autokeras/image_supervised.py", line 40, in run_searcher_once
    searcher.search(train_data, test_data, timeout)
  File "/home/monika/.local/lib/python3.6/site-packages/autokeras/search.py", line 190, in search
    timeout)
  File "/home/monika/.local/lib/python3.6/site-packages/autokeras/bayesian.py", line 257, in optimize_acq
    for temp_graph in transform(elem.graph):
  File "/home/monika/.local/lib/python3.6/site-packages/autokeras/net_transformer.py", line 83, in transform
    temp_graph = to_deeper_graph(deepcopy(graph))
  File "/home/monika/.local/lib/python3.6/site-packages/autokeras/net_transformer.py", line 63, in to_deeper_graph
    graph.to_conv_deeper_model(layer_id, 3)
  File "/home/monika/.local/lib/python3.6/site-packages/autokeras/graph.py", line 313, in to_conv_deeper_model
    new_layers = deeper_conv_block(target, kernel_size, self.weighted)
  File "/home/monika/.local/lib/python3.6/site-packages/autokeras/layer_transformer.py", line 11, in deeper_conv_block
    weight = np.zeros((n_filters, n_filters) + filter_shape)
MemoryError

 If I run the code with the mnist classification example, I get the following error after the 8th model :+1: 
                                                                           Process SpawnPoolWorker-9:
Traceback (most recent call last):                                         
  File "/usr/lib/python3.6/multiprocessing/pool.py", line 125, in worker
    put((job, i, result))
  File "/usr/lib/python3.6/multiprocessing/queues.py", line 347, in put
    self._writer.send_bytes(obj)
  File "/usr/lib/python3.6/multiprocessing/connection.py", line 200, in send_bytes
    self._send_bytes(m[offset:offset + size])
  File "/usr/lib/python3.6/multiprocessing/connection.py", line 397, in _send_bytes
    self._send(header)
  File "/usr/lib/python3.6/multiprocessing/connection.py", line 368, in _send
    n = write(self._handle, buf)
BrokenPipeError: [Errno 32] Broken pipe

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
    self.run()
  File "/usr/lib/python3.6/multiprocessing/process.py", line 93, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/lib/python3.6/multiprocessing/pool.py", line 130, in worker
    put((job, i, (False, wrapped)))
  File "/usr/lib/python3.6/multiprocessing/queues.py", line 347, in put
    self._writer.send_bytes(obj)
  File "/usr/lib/python3.6/multiprocessing/connection.py", line 200, in send_bytes
    self._send_bytes(m[offset:offset + size])
  File "/usr/lib/python3.6/multiprocessing/connection.py", line 404, in _send_bytes
    self._send(header + buf)
  File "/usr/lib/python3.6/multiprocessing/connection.py", line 368, in _send
    n = write(self._handle, buf)
BrokenPipeError: [Errno 32] Broken pipe
/usr/lib/python3.6/multiprocessing/semaphore_tracker.py:143: UserWarning: semaphore_tracker: There appear to be 1 leaked semaphores to clean up at shutdown
  len(cache))
/usr/lib/python3.6/multiprocessing/semaphore_tracker.py:143: UserWarning: semaphore_tracker: There appear to be 4 leaked semaphores to clean up at shutdown
  len(cache))


### Reproducing Steps
Steps to reproduce the behavior:
  * Step 1: ... use the mnist.py from examples. 
  * Step 2: ... run the script with either the custom dataset for classification or the default example

### Expected Behavior
<!---
A clear and concise description of what you expected to happen.
-->

### Setup Details
Include the details about the versions of:
 - OS type and version: Ubuntu 18.04
 - Python: 3.6.5
 - autokeras: <!--- e.g. 0.2.1, 0.2.0, master-->0.2.14
 - scikit-learn:0.19.1
 - numpy:1.15.2
 - keras:2.2.2
 - scipy:1.1.0
 - tensorflow:1.10.1
 - pytorch:0.4.1

### Additional context

My data is of the following dimension : 
Training Data : (42532, 128,128), training label :  ( 42532,24)
Testing Data : ( 1325,128,128) , testing label : (1325,24)
Running the code on GTX 1080TI 11GB and 32GB RAM




Discussion:
I met the same error
```
Traceback (most recent call last):
  File "ak_aoi_img.py", line 23, in <module>
    clf.fit(x_train, y_train, time_limit=2 * 60 * 60)
  File "/home/adt/wen/ak_aoi/autokeras/image_supervised.py", line 239, in fit
    run_searcher_once(train_data, test_data, self.path, int(time_remain))
  File "/home/adt/wen/ak_aoi/autokeras/image_supervised.py", line 40, in run_searcher_once
    searcher.search(train_data, test_data, timeout)
  File "/home/adt/wen/ak_aoi/autokeras/search.py", line 193, in search
    remaining_time)
  File "/home/adt/wen/ak_aoi/autokeras/bayesian.py", line 256, in optimize_acq
    for temp_graph in transform(elem.graph):
  File "/home/adt/wen/ak_aoi/autokeras/net_transformer.py", line 83, in transform
    temp_graph = to_deeper_graph(deepcopy(graph))
  File "/home/adt/wen/ak_aoi/autokeras/net_transformer.py", line 63, in to_deeper_graph
    graph.to_conv_deeper_model(layer_id, 3)
  File "/home/adt/wen/ak_aoi/autokeras/graph.py", line 313, in to_conv_deeper_model
    new_layers = deeper_conv_block(target, kernel_size, self.weighted)
  File "/home/adt/wen/ak_aoi/autokeras/layer_transformer.py", line 11, in deeper_conv_block
    weight = np.zeros((n_filters, n_filters) + filter_shape)
MemoryError
```
The solution would be to load the data from disk batch by batch, which can be done by pytorch dataloader.


Resolving patch:
diff --git a/autokeras/image/image_supervised.py b/autokeras/image/image_supervised.py
--- a/autokeras/image/image_supervised.py
+++ b/autokeras/image/image_supervised.py
@@ -1,4 +1,3 @@
-import csv
 import os
 import pickle
 from abc import abstractmethod
@@ -6,7 +5,6 @@
 
 import numpy as np
 import torch
-from scipy import ndimage
 from sklearn.model_selection import train_test_split
 
 from autokeras.cnn_module import CnnModule
@@ -15,28 +13,8 @@
 from autokeras.nn.metric import Accuracy, MSE
 from autokeras.preprocessor import OneHotEncoder, ImageDataTransformer
 from autokeras.supervised import Supervised, PortableClass
-from autokeras.utils import has_file, pickle_from_file, pickle_to_file, temp_folder_generator, validate_xy
-
-
-def read_csv_file(csv_file_path):
-    """Read the csv file and returns two separate list containing files name and their labels.
-
-    Args:
-        csv_file_path: Path to the CSV file.
-
-    Returns:
-        file_names: List containing files names.
-        file_label: List containing their respective labels.
-    """
-    file_names = []
-    file_labels = []
-    with open(csv_file_path, 'r') as files_path:
-        path_list = csv.DictReader(files_path)
-        fieldnames = path_list.fieldnames
-        for path in path_list:
-            file_names.append(path[fieldnames[0]])
-            file_labels.append(path[fieldnames[1]])
-    return file_names, file_labels
+from autokeras.utils import has_file, pickle_from_file, pickle_to_file, temp_folder_generator, validate_xy, \
+    read_csv_file, read_image
 
 
 def read_images(img_file_names, images_dir_path):
@@ -52,7 +30,7 @@ def read_images(img_file_names, images_dir_path):
         for img_file in img_file_names:
             img_path = os.path.join(images_dir_path, img_file)
             if os.path.exists(img_path):
-                img = ndimage.imread(fname=img_path)
+                img = read_image(img_path)
                 if len(img.shape) < 3:
                     img = img[..., np.newaxis]
                 x_train.append(img)
diff --git a/autokeras/preprocessor.py b/autokeras/preprocessor.py
--- a/autokeras/preprocessor.py
+++ b/autokeras/preprocessor.py
@@ -1,9 +1,11 @@
+import os
 import numpy as np
 import torch
 from torch.utils.data import Dataset, DataLoader
 from torchvision.transforms import ToPILImage, RandomCrop, RandomHorizontalFlip, ToTensor, Normalize, Compose
 from abc import ABC, abstractmethod
 from autokeras.constant import Constant
+from autokeras.utils import read_csv_file, read_image
 
 
 class OneHotEncoder:
@@ -84,7 +86,7 @@ def _transform(self, compose_list, data, targets):
         return MultiTransformDataset(data, targets, data_transforms)
 
 
-class ImageDataTransformer:
+class ImageDataTransformer(DataTransformer):
     def __init__(self, data, augment=Constant.DATA_AUGMENTATION):
         self.max_val = data.max()
         data = data / self.max_val
@@ -145,3 +147,28 @@ def __getitem__(self, index):
 
     def __len__(self):
         return len(self.dataset)
+
+
+class BatchDataset(Dataset):
+    def __init__(self, csv_file_path, image_path, has_target=True):
+        file_names, target = read_csv_file(csv_file_path)
+
+        self.y_encoder = OneHotEncoder()
+        self.y_encoder.fit(target)
+        target = self.y_encoder.transform(target)
+
+        self.target = target
+        self.has_target = has_target
+        self.file_paths = list(map(lambda file_name: os.path.join(image_path, file_name), file_names))
+
+    def __getitem__(self, index):
+        image = read_image(self.file_paths[index])
+        if len(image.shape) < 3:
+            image = image[..., np.newaxis]
+        image = torch.Tensor(image.transpose(2, 0, 1))
+        if self.has_target:
+            return image, self.target[index]
+        return image
+
+    def __len__(self):
+        return len(self.file_paths)
diff --git a/autokeras/utils.py b/autokeras/utils.py
--- a/autokeras/utils.py
+++ b/autokeras/utils.py
@@ -1,3 +1,4 @@
+import csv
 import os
 import pickle
 import sys
@@ -6,6 +7,7 @@
 
 import requests
 import torch
+from scipy import ndimage
 
 from autokeras.constant import Constant
 
@@ -169,3 +171,29 @@ def validate_xy(x_train, y_train):
 
     if x_train.shape[0] != y_train.shape[0]:
         raise ValueError('x_train and y_train should have the same number of instances.')
+
+
+def read_csv_file(csv_file_path):
+    """Read the csv file and returns two separate list containing files name and their labels.
+
+    Args:
+        csv_file_path: Path to the CSV file.
+
+    Returns:
+        file_names: List containing files names.
+        file_label: List containing their respective labels.
+    """
+    file_names = []
+    file_labels = []
+    with open(csv_file_path, 'r') as files_path:
+        path_list = csv.DictReader(files_path)
+        fieldnames = path_list.fieldnames
+        for path in path_list:
+            file_names.append(path[fieldnames[0]])
+            file_labels.append(path[fieldnames[1]])
+    return file_names, file_labels
+
+
+def read_image(img_path):
+    img = ndimage.imread(fname=img_path)
+    return img
\ No newline at end of file


Test patch:
diff --git a/tests/common.py b/tests/common.py
--- a/tests/common.py
+++ b/tests/common.py
@@ -262,3 +262,8 @@ def map_async(self, a, b):
 def simple_transform(graph):
     graph.to_wider_model(5, 64)
     return [deepcopy(graph)]
+
+
+def mock_train(**kwargs):
+    str(kwargs)
+    return 1, 0
\ No newline at end of file
diff --git a/tests/image/test_image_supervised.py b/tests/image/test_image_supervised.py
--- a/tests/image/test_image_supervised.py
+++ b/tests/image/test_image_supervised.py
@@ -3,12 +3,7 @@
 import pytest
 
 from autokeras.image.image_supervised import *
-from tests.common import clean_dir, MockProcess, simple_transform
-
-
-def mock_train(**kwargs):
-    str(kwargs)
-    return 1, 0
+from tests.common import clean_dir, MockProcess, simple_transform, mock_train
 
 
 def test_train_x_array_exception():
@@ -175,12 +170,6 @@ def test_fit_csv_file(_, _1):
     clean_dir(os.path.join(path, "temp"))
 
 
-@patch('autokeras.image.image_supervised.temp_folder_generator', return_value='dummy_path/')
-def test_init_image_classifier_with_none_path(_):
-    clf = ImageClassifier()
-    assert clf.path == 'dummy_path/'
-
-
 @patch('torch.multiprocessing.Pool', new=MockProcess)
 @patch('autokeras.search.ModelTrainer.train_model', side_effect=mock_train)
 def test_fit_predict_regression(_):
diff --git a/tests/test_preprocessor.py b/tests/test_preprocessor.py
new file mode 100644
--- /dev/null
+++ b/tests/test_preprocessor.py
@@ -0,0 +1,28 @@
+from unittest.mock import patch
+
+from autokeras.cnn_module import CnnModule
+from autokeras.nn.loss_function import classification_loss
+from autokeras.nn.metric import Accuracy
+from autokeras.preprocessor import *
+from tests.common import MockProcess, mock_train, clean_dir
+
+
+@patch('torch.multiprocessing.Pool', new=MockProcess)
+@patch('autokeras.search.ModelTrainer.train_model', side_effect=mock_train)
+def test_batch_dataset(_):
+    Constant.MAX_ITER_NUM = 1
+    Constant.MAX_MODEL_NUM = 4
+    Constant.SEARCH_MAX_ITER = 1
+    Constant.T_MIN = 0.8
+    data_path = 'tests/resources'
+    path = 'tests/resources/temp'
+    clean_dir(path)
+    csv_file_path = os.path.join(data_path, "images_test/images_name.csv")
+    image_path = os.path.join(data_path, "images_test/Color_images")
+    train_dataset = BatchDataset(csv_file_path, image_path, has_target=True)
+    test_dataset = BatchDataset(csv_file_path, image_path, has_target=True)
+    train_dataloader = DataLoader(train_dataset, batch_size=1, shuffle=False)
+    test_dataloader = DataLoader(test_dataset, batch_size=1, shuffle=False)
+    cnn = CnnModule(classification_loss, Accuracy, {}, path, True)
+    cnn.fit(2, (4, 250, 250, 3), train_dataloader, test_dataloader, 12*60*60)
+    clean_dir(path)