Issue 1:
Hi,

On a Google Colab notebook with keras(2.2.4) and tensorflow(1.13.1) as a backend, I am trying to tune a CNN, I use a simple and basic table of hyper-parameters and run my tests in a set of loops.

My problem is that I can't free the GPU memory after each iteration and Keras doesn't seem to be able to release GPU memory automatically. So every time I get a Ressource Exhausted : Out Of Memory (OOM)

I did some digging up and run into this function that reassembles different solutions that have been suggested to solve this problem (didn't work for me though) :

def reset_keras():
sess = get_session()
clear_session()
sess.close()
sess = get_session()

try:
del model # this is from global space - change this as you need
except:
pass

print(gc.collect()) # if it's done something you should see a number being outputted

# use the same config as you used to create the session
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 1
config.gpu_options.visible_device_list = "0"
set_session(tf.Session(config=config))

The only thing that i didn't fully grasp is the "same config as you used to create your model " since with Keras we don't chose explicitly a certain configuration. I get by for one iteration, some times two, but I can't go beyond. I already tried to change the batch_size and for the moment I am unable to afford for a machine with higher performances.

Issue 2:
In order to preform a object detection like task, I used a CNN. I am using a custom generator that follows this example

My machine is a MacOS High Sierra 10.13.6, yet I am running my code using Google Colab's GPU so my machine's performances has nothing to do with the issue(right ?). It should be noted that I have tested my code on another platform (FloydHub's Tesla K80 GPU) and had the same problem.

I am using

Keras 2.2.4

Tensorflow backend 1.13.1

Python (3.7)

CUDA release 10.1, V10.1.168

GPU model and memory: Tesla K80 (up to my knowledge, this is what google uses on Colab)

I run out of GPU memory quickly, when passing my model's training in a for loop (looking for optimal hyper-parameters) the program stops on Out Of Memory Error after the first iteration. Before the beginning of the first epoch and shortly before the loading arrow for the first epoch appears, the GPU usage jumps to a value of 98%. I put a gc.collect() , del model and K.clear_session() in the end of my training in order to release memory for the next model but it seems to have no effect on GPU which remains at 98% before eventually crashing my notebook.

---------------------------------------------------------------------------
ResourceExhaustedError Traceback (most recent call last)
<ipython-input-75-31b4f9e0b883> in <module>()
15 models={}
16 compile_model(model,OPTIMIZER,LOSS,METRICS,LR,MOMENTUM,DECAY)
---> 17 allinone2(Hist,model,Data,VAL_SPLIT,OPTIMIZER,LOSS,METRICS,LR,MOMENTUM,DECAY,BATCH_SIZE,EPOCHS,PARAMS,True)

9 frames
<ipython-input-71-125d04498229> in allinone2(Hist, model, Data, validation_split, optimizer, loss, metric, lr, momentum, decay, batch_size, epochs, params, showbboxs)
2 if TPU :
3 model = tf.contrib.tpu.keras_to_tpu_model(model,strategy=tf.contrib.tpu.TPUDistributionStrategy(tf.contrib.cluster_resolver.TPUClusterResolver(TPU_WORKER)))
----> 4 train_gen,validation_gen,Hist = fit(model,Hist,Data,VAL_SPLIT,BATCH_SIZE,EPOCHS,PARAMS)
5 Save_model(SAVE_DIR,name_json="model",nameh5="model",save=SAVE_MODEL)
6 training_results(Hist,str(validation_split)+str(batch_size)+str(epochs),savefig=SAVE_FIG)

<ipython-input-64-efd2c5419ad6> in fit(model, Hist, Data, validation_split, batch_size, epochs, params)
17 validation_steps = lenvalidation // batch_size,
18 callbacks=[Hist],
---> 19 validation_data = validation_gen
20 )
21

/usr/local/lib/python3.6/dist-packages/keras/legacy/interfaces.py in wrapper(*args, **kwargs)
89 warnings.warn('Update your `' + object_name + '` call to the ' +
90 'Keras 2 API: ' + signature, stacklevel=2)
---> 91 return func(*args, **kwargs)
92 wrapper._original_function = func
93 return wrapper

/usr/local/lib/python3.6/dist-packages/keras/engine/training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
1416 use_multiprocessing=use_multiprocessing,
1417 shuffle=shuffle,
-> 1418 initial_epoch=initial_epoch)
1419
1420 @interfaces.legacy_generator_methods_support

/usr/local/lib/python3.6/dist-packages/keras/engine/training_generator.py in fit_generator(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
215 outs = model.train_on_batch(x, y,
216 sample_weight=sample_weight,
--> 217 class_weight=class_weight)
218
219 outs = to_list(outs)

/usr/local/lib/python3.6/dist-packages/keras/engine/training.py in train_on_batch(self, x, y, sample_weight, class_weight)
1215 ins = x + y + sample_weights
1216 self._make_train_function()
-> 1217 outputs = self.train_function(ins)
1218 return unpack_singleton(outputs)
1219

/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py in __call__(self, inputs)
2713 return self._legacy_call(inputs)
2714
-> 2715 return self._call(inputs)
2716 else:
2717 if py_any(is_tensor(x) for x in inputs):

/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py in _call(self, inputs)
2673 fetched = self._callable_fn(*array_vals, run_metadata=self.run_metadata)
2674 else:
-> 2675 fetched = self._callable_fn(*array_vals)
2676 return fetched[:len(self.outputs)]
2677

/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py in __call__(self, *args, **kwargs)
1437 ret = tf_session.TF_SessionRunCallable(
1438 self._session._session, self._handle, args, status,
-> 1439 run_metadata_ptr)
1440 if run_metadata:
1441 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)

/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/errors_impl.py in __exit__(self, type_arg, value_arg, traceback_arg)
526 None, None,
527 compat.as_text(c_api.TF_Message(self.status.status)),
--> 528 c_api.TF_GetCode(self.status.status))
529 # Delete the underlying status object from memory otherwise it stays alive
530 # as there is a reference to status from this from the traceback due to

ResourceExhaustedError: OOM when allocating tensor with shape[3114176,32] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc
[[{{node training_3/Adam/gradients/dense_17/MatMul_grad/MatMul_1}}]]
Hint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.

I should be able to free the GPU memory in order to run my tests, It should be noted that my dataset has 1000 images of 800x1000 shape.

A snippet of the code that causes the problem (.ipynb => .py) :

!pip install gputil
!pip install psutil
!pip install humanize
#Imports ...

def penalty(y_true, y_pred):
a = tf.placeholder(tf.float64)
p = 20
loss = K.switch(tf.less(y_true - y_pred, 0.1), K.square(y_true - y_pred) , p * K.square(y_true - y_pred))
return K.max(loss)

def cusloss(y_true,y_pred):
return K.mean(K.equal(K.round(100*y_pred , 100*y_true)),tf.zeros_like(y_pred))

# Global parameters
IMG_SIZE = 800,1000,3
BATCH_SIZEs = [4,8,16,32]
EPOCHSs = [10]
PARAMSs = [{"rotation":20,"scale":0.1,"shear":0.1,"translate":0.1,"scale":0.1}]
OPTIMIZERs = ['adam']
LOSSEs = [penalty]
METRICSs = [[cusloss,penalty,losses.msle,losses.mae,losses.mse,losses.logcosh]]
LRs = [0.01,0.005]
MOMENTUMs = [0.05,0.1]
DECAYs = [0.2]
VAL_SPLITs = [0.2]

def load_data(*args):
...
return Data

class Generator(keras.utils.Sequence):

def __init__(self, Data, params ,batch_size):
"""Initials an image generator"""
self.x = col(Data,0,1)
self.y = np.array(col(Data,1,5))
self.batch_size = batch_size
self.params = params
self.on_epoch_end()

def __len__(self):
"""Our generator's length"""
return int(np.ceil(len(self.x) / float(self.batch_size)))

def __getitem__(self, idx):
""" __getitem__ gives access to an item of our generator which will be a batch of BATCH_SIZE size. It is called via the brackets []"""
images_list = self.x
bboxes = self.y
batch_size = self.batch_size
w,h = IMG_SIZE[:2]
required_transformations = self.params.keys()

# Generate indexes of the batch
indexes = self.indexes[idx*self.batch_size:(idx+1)*self.batch_size]
if sum(indexes) == 0 : # check if indexes is empty
return None

# extracting BATCH_SIZE images from the dataset
x_temp = [self.x[i] for i in indexes]
y_temp = np.array([self.y[i,] for i in indexes])

# shuffling our batch's elements
Ids,y0 = sklearn.utils.shuffle(x_temp,y_temp)

# Initiate our batchs
X = np.zeros((batch_size,h,w,1))
y = np.zeros((batch_size,4))

for i,image in enumerate(Ids) :
image = image[0]
try :
img = cv2.imread("scans/"+image+".png")
except :
print("Unable to read %s"%(image))
img = None

# Starting preprocessing
if not (img is None) :
img = cv2.resize(img,(w,h))
bbox = y0[i,]
try :
X[i,:,:,:] = np.expand_dims(rgb2gray(img_),axis=2)
y[i,:] = bboxes_

except Exception as exc:
# Prints the error without stoping the process
print("Error found at index %d"%(i))
print(traceback.format_exc())
print(exc)

else :
print("%s is None"%(image))
return X,y

def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.x))
np.random.shuffle(self.indexes)

def fit(model,Hist,Data,validation_split,batch_size,epochs,params):
""" Fits our input to the output"""
n = len(Data)
lenvalidation = int(validation_split*n)
lentrain = n - lenvalidation
train_set = sklearn.utils.shuffle(Data[:lentrain])
validation_set = sklearn.utils.shuffle(Data[lentrain:])
train_gen = Generator(train_set,params,batch_size)
validation_gen = Generator(validation_set,params,batch_size)

print("starting training...")
model.fit_generator(train_gen,
steps_per_epoch = lentrain // batch_size,
epochs=epochs,
validation_steps = lenvalidation // batch_size,
callbacks=[Hist],
validation_data = validation_gen
)

return train_gen,validation_gen,Hist

def create_model():
# History initialisation
Hist = History()
# Defining the architecture of our CNN based NN
model = Sequential()
model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=(IMG_SIZE[1],IMG_SIZE[0],1)))
model.add(MaxPooling2D(pool_size=(2, 2),strides=(2,2)))
model.add(Dropout(0.25))
model.add(Conv2D(64, kernel_size=(5, 5), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2),strides=(2,2)))
model.add(Dropout(0.3))
model.add(Flatten())
model.add(Dense(32, activation='relu'))
model.add(Dropout(0.3))
model.add(Dense(4))
return Hist,model

def compile_model(model,optimize,loss,metric,lr,momentum,decay):
# Compiles the model
run_opts = tf.RunOptions(report_tensor_allocations_upon_oom = True)
model.compile(optimizer=optimize , loss = loss, metrics = metric, options = run_opts)

# Define optimizers
K.set_value(model.optimizer.lr, lr)
K.set_value(model.optimizer.decay, decay)
try :
K.set_value(optimizer.momentum, momentum)
except :
pass

def allinone(Hist,model,Data,validation_split,optimizer,loss,metric,lr,momentum,decay,batch_size,epochs,params) :
compile_model(model,optimizer,loss,metric,lr,momentum,decay)
train_gen,validation_gen,Hist = fit(model,Hist,Data,VAL_SPLIT,BATCH_SIZE,EPOCHS,PARAMS)
torch.cuda.empty_cache()
del model
gc.collect()
K.clear_session()

def printmm(y_true=0,y_pred=0):
GPUs = GPUtil.getGPUs()
gpu = GPUs[0]
print(GPUtil.showUtilization())
return(tf.Variable(gpu.memoryUtil*100))

# The Ultimate test
Data = load_data()
Hist,model = create_model()
for EPOCHS in EPOCHSs:
for BATCH_SIZE in BATCH_SIZEs:
for PARAMS in PARAMSs :
for OPTIMIZER in OPTIMIZERs:
for LOSS in LOSSEs :
for METRICS in METRICSs:
for LR in LRs :
for MOMENTUM in MOMENTUMs:
for DECAY in DECAYs:
for VAL_SPLIT in VAL_SPLITs :
allinone(Hist,model,Data,VAL_SPLIT,OPTIMIZER,LOSS,METRICS,LR,MOMENTUM,DECAY,BATCH_SIZE,EPOCHS,PARAMS)