Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<compute_train_metric>
%matplotlib inline
np.random.seed(2)
 sns.set(style='white', context='notebook', palette='deep' )
def get_iou_vector(A, B):
batch_size = A.shape[0]
metric = 0.0
for batch in range(batch_size):
t, p = A[batch], B[batch]
true = np.sum(t)
 pred = np.sum(p)
 if true == 0:
metric +=(pred == 0)
 continue
intersection = np.sum(t * p)
 union = true + pred - intersection
iou = intersection / union
iou = np.floor(max(0,(iou - 0.45)*20)) / 10
metric += iou
metric /= batch_size
return metric
def my_iou_metric(label, pred):
return tf.py_func(get_iou_vector, [label, pred > 0.5], tf.float64 )<compute_test_metric>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 Y_train = train["label"]
X_train = train.drop(labels = ["label"], axis = 1 )
def dice_coef(y_true, y_pred):
y_true_f = K.flatten(y_true)
 y_pred = K.cast(y_pred, 'float32')
 y_pred_f = K.cast(K.greater(K.flatten(y_pred), 0.5), 'float32')
 intersection = y_true_f * y_pred_f
score = 2.* K.sum(intersection)/(K.sum(y_true_f)+ K.sum(y_pred_f))
return score
def dice_loss(y_true, y_pred):
smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = y_true_f * y_pred_f
score =(2.* K.sum(intersection)+ smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 return 1.- score
def bce_dice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)+ dice_loss(y_true, y_pred)
 def bce_logdice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)- K.log(1.- dice_loss(y_true, y_pred))<choose_model_class>
( x_train1, y_train1),(x_test1, y_test1)= mnist.load_data()
train1 = np.concatenate([x_train1, x_test1], axis=0)
 y_train1 = np.concatenate([y_train1, y_test1], axis=0)
 Y_train1 = y_train1
X_train1 = train1.reshape(-1, 28*28 )
class SnapshotCallbackBuilder:
def __init__(self, nb_epochs, nb_snapshots, init_lr=0.1):
self.T = nb_epochs
self.M = nb_snapshots
self.alpha_zero = init_lr
def get_callbacks(self, model_prefix='Model'):
callback_list = [
callbacks.ModelCheckpoint("./keras.model",monitor='val_my_iou_metric',
mode = 'max', save_best_only=True, verbose=1),
swa,
callbacks.LearningRateScheduler(schedule=self._cosine_anneal_schedule)
 ]
return callback_list
def _cosine_anneal_schedule(self, t):
cos_inner = np.pi *(t %(self.T // self.M))
cos_inner /= self.T // self.M
cos_out = np.cos(cos_inner)+ 1
return float(self.alpha_zero / 2 * cos_out )<define_search_model>
X_train = X_train / 255.0
test = test / 255.0
X_train1 = X_train1 / 255.0
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 x = BatchNormalization()(x)
 if activation == True:
x = LeakyReLU(alpha=0.1 )(x)
 return x
def residual_block(blockInput, num_filters=16):
x = LeakyReLU(alpha=0.1 )(blockInput)
 x = BatchNormalization()(x)
 blockInput = BatchNormalization()(blockInput)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False)
 x = Add()([x, blockInput])
 return x<choose_model_class>
X_train = np.concatenate(( X_train.values, X_train1))
Y_train = np.concatenate(( Y_train, Y_train1))
def UXception(input_shape=(None, None, 3)) :
backbone = Xception(input_shape=input_shape,weights='imagenet',include_top=False)
 input = backbone.input
start_neurons = 16
conv4 = backbone.layers[121].output
conv4 = LeakyReLU(alpha=0.1 )(conv4)
 pool4 = MaxPooling2D(( 2, 2))(conv4)
 pool4 = Dropout(0.1 )(pool4)
 convm = Conv2D(start_neurons * 32,(3, 3), activation=None, padding="same" )(pool4)
 convm = residual_block(convm,start_neurons * 32)
 convm = residual_block(convm,start_neurons * 32)
 convm = LeakyReLU(alpha=0.1 )(convm)
 deconv4 = Conv2DTranspose(start_neurons * 16,(3, 3), strides=(2, 2), padding="same" )(convm)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(0.1 )(uconv4)
 uconv4 = Conv2D(start_neurons * 16,(3, 3), activation=None, padding="same" )(uconv4)
 uconv4 = residual_block(uconv4,start_neurons * 16)
 uconv4 = residual_block(uconv4,start_neurons * 16)
 uconv4 = LeakyReLU(alpha=0.1 )(uconv4)
 deconv3 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(uconv4)
 conv3 = backbone.layers[31].output
uconv3 = concatenate([deconv3, conv3])
 uconv3 = Dropout(0.1 )(uconv3)
 uconv3 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(uconv3)
 uconv3 = residual_block(uconv3,start_neurons * 8)
 uconv3 = residual_block(uconv3,start_neurons * 8)
 uconv3 = LeakyReLU(alpha=0.1 )(uconv3)
 deconv2 = Conv2DTranspose(start_neurons * 4,(3, 3), strides=(2, 2), padding="same" )(uconv3)
 conv2 = backbone.layers[21].output
conv2 = ZeroPadding2D(((1,0),(1,0)) )(conv2)
 uconv2 = concatenate([deconv2, conv2])
 uconv2 = Dropout(0.1 )(uconv2)
 uconv2 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(uconv2)
 uconv2 = residual_block(uconv2,start_neurons * 4)
 uconv2 = residual_block(uconv2,start_neurons * 4)
 uconv2 = LeakyReLU(alpha=0.1 )(uconv2)
 deconv1 = Conv2DTranspose(start_neurons * 2,(3, 3), strides=(2, 2), padding="same" )(uconv2)
 conv1 = backbone.layers[11].output
conv1 = ZeroPadding2D(((3,0),(3,0)) )(conv1)
 uconv1 = concatenate([deconv1, conv1])
 uconv1 = Dropout(0.1 )(uconv1)
 uconv1 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(uconv1)
 uconv1 = residual_block(uconv1,start_neurons * 2)
 uconv1 = residual_block(uconv1,start_neurons * 2)
 uconv1 = LeakyReLU(alpha=0.1 )(uconv1)
 uconv0 = Conv2DTranspose(start_neurons * 1,(3, 3), strides=(2, 2), padding="same" )(uconv1)
 uconv0 = Dropout(0.1 )(uconv0)
 uconv0 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(uconv0)
 uconv0 = residual_block(uconv0,start_neurons * 1)
 uconv0 = residual_block(uconv0,start_neurons * 1)
 uconv0 = LeakyReLU(alpha=0.1 )(uconv0)
 uconv0 = Dropout(0.1/2 )(uconv0)
 output_layer = Conv2D(1,(1,1), padding="same", activation="sigmoid" )(uconv0)
 model = Model(input, output_layer)
 model.name = 'u-xception'
return model<choose_model_class>
Y_train = to_categorical(Y_train, num_classes = 10 )
K.clear_session()
model = UXception(input_shape=(img_size_target,img_size_target,3))<find_best_params>
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state=2 )
def l2_norm(grad):
norm = K.sqrt(K.sum(K.square(grad)))+ K.epsilon()
return norm
class OptimizerWrapper(optimizers.Optimizer):
def __init__(self, optimizer):
self.optimizer = optimizers.get(optimizer)
 self._optimizer_get_gradients = self.optimizer.get_gradients
def get_gradients(self, loss, params):
grads = self._optimizer_get_gradients(loss, params)
 return grads
@interfaces.legacy_get_updates_support
def get_updates(self, loss, params):
self.optimizer.get_gradients = self.get_gradients
self.optimizer.get_updates(loss, params)
 self.optimizer.get_gradients = self._optimizer_get_gradients
return self.updates
def set_weights(self, weights):
self.optimizer.set_weights(weights)
 def get_weights(self):
return self.optimizer.get_weights()
def get_config(self):
config = {'optimizer_name': self.optimizer.__class__.__name__.lower() }
optimizer_config = {'optimizer_config': self.optimizer.get_config() }
return dict(list(optimizer_config.items())+ list(config.items()))
@property
def weights(self):
return self.optimizer.weights
@property
def updates(self):
return self.optimizer.updates
@classmethod
def from_config(cls, config):
raise NotImplementedError
@classmethod
def set_normalization_function(cls, name, func):
global _NORMS
_NORMS[name] = func
@classmethod
def get_normalization_functions(cls):
global _NORMS
return sorted(list(_NORMS.keys()))
class NormalizedOptimizer(OptimizerWrapper):
def __init__(self, optimizer, normalization='l2'):
super(NormalizedOptimizer, self ).__init__(optimizer)
 if normalization not in _NORMS:
raise ValueError('`normalization` must be one of %s.
 '
'Provided was "%s".' %(str(sorted(list(_NORMS.keys()))), normalization))
self.normalization = normalization
self.normalization_fn = _NORMS[normalization]
self.lr = K.variable(1e-3, name='lr')
 def get_gradients(self, loss, params):
grads = super(NormalizedOptimizer, self ).get_gradients(loss, params)
 grads = [grad / self.normalization_fn(grad)for grad in grads]
return grads
def get_config(self):
config = {'normalization': self.normalization}
base_config = super(NormalizedOptimizer, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
@classmethod
def from_config(cls, config):
optimizer_config = {'class_name': config['optimizer_name'],
'config': config['optimizer_config']}
optimizer = optimizers.get(optimizer_config)
 normalization = config['normalization']
return cls(optimizer, normalization=normalization)
 _NORMS = {
'l2': l2_norm,
}
get_custom_objects().update({'NormalizedOptimizer': NormalizedOptimizer} )<train_model>

 model = Sequential()
model.add(Conv2D(filters = 64, kernel_size =(5,5),padding = 'Same', activation ='relu', input_shape =(28,28,1)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters = 64, kernel_size =(5,5),padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3), padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(BatchNormalization())
 model.add(Dropout(0.25))
model.add(Dense(10, activation = "softmax"))
class SWA(keras.callbacks.Callback):
def __init__(self, filepath, swa_epoch):
super(SWA, self ).__init__()
self.filepath = filepath
self.swa_epoch = swa_epoch
def on_train_begin(self, logs=None):
self.nb_epoch = self.params['epochs']
print('Stochastic weight averaging selected for last {} epochs.' 
.format(self.nb_epoch - self.swa_epoch))
def on_epoch_end(self, epoch, logs=None):
if epoch == self.swa_epoch:
self.swa_weights = self.model.get_weights()
elif epoch > self.swa_epoch:
for i in range(len(self.swa_weights)) :
self.swa_weights[i] =(self.swa_weights[i] * 
(epoch - self.swa_epoch)+ self.model.get_weights() [i])/(( epoch - self.swa_epoch)+ 1)
 else:
pass
def on_train_end(self, logs=None):
self.model.set_weights(self.swa_weights)
 print('Final model parameters set to stochastic weight average.')
 self.model.save_weights(self.filepath)
 print('Final stochastic averaged weights saved to file.' )<choose_model_class>
plot_model(model, to_file='model.png', show_shapes=True, show_layer_names=True)
 Image("model.png" )
sgd = SGD(0.01, momentum=0.9, nesterov=True)
 sgd = NormalizedOptimizer(sgd, normalization='l2' )<choose_model_class>
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0 )
model.compile(loss=bce_dice_loss, optimizer=sgd, metrics=[my_iou_metric] )<concatenate>
model.compile(optimizer = optimizer , loss = "categorical_crossentropy", metrics=["accuracy"] )
x_train = np.append(x_train, [np.fliplr(x)for x in x_train[0:3200]], axis=0)
 y_train = np.append(y_train, [np.fliplr(x)for x in y_train[0:3200]], axis=0)
 x_train = np.repeat(x_train,3,axis=3)
 x_valid = np.repeat(x_valid,3,axis=3 )<train_model>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )
epochs = 40
snapshot = SnapshotCallbackBuilder(nb_epochs=epochs,nb_snapshots=1,init_lr=1e-3)
 batch_size = 32
swa = SWA('./keras_swa.model',35)
 history = model.fit(x_train, y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=snapshot.get_callbacks() ,shuffle=True,verbose=2 )<prepare_x_and_y>
epochs = 50
batch_size = 128
del ids_train, x_train, y_train,some_salt_ids,depths_df
gc.collect()<load_pretrained>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )
try:
print('using swa weight model')
 model.load_weights('./keras_swa.model')
 except:
model.load_weights('./keras.model' )<predict_on_test>
history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val,Y_val),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size
, callbacks=[learning_rate_reduction] )
def predict_result(model,x_test,img_size_target,batch_size):
preds_test1 = model.predict([x_test],batch_size=batch_size ).reshape(-1, img_size_target, img_size_target)
 x_test_reflect2 = np.array([np.fliplr(x)for x in x_test])
 preds_test2_refect = model.predict([x_test_reflect2],batch_size=batch_size ).reshape(-1, img_size_target, img_size_target)
 preds_test2 = np.array([ np.fliplr(x)for x in preds_test2_refect])
 x_test_reflect3 = np.array([np.flipud(x)for x in x_test])
 preds_test3_refect = model.predict([x_test_reflect3],batch_size=batch_size ).reshape(-1, img_size_target, img_size_target)
 preds_test3 = np.array([ np.flipud(x)for x in preds_test3_refect])
 x_test_reflect4 = np.array([np.fliplr(np.flipud(x)) for x in x_test])
 preds_test4_refect = model.predict([x_test_reflect4],batch_size=batch_size ).reshape(-1, img_size_target, img_size_target)
 preds_test4 = np.array([ np.fliplr(np.flipud(x)) for x in preds_test4_refect])
 preds_avg =(preds_test1+preds_test2+preds_test3+preds_test4)/4
return preds_avg<predict_on_test>
results = model.predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )
<compute_train_metric><EOS> 
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_submission.csv",index=False )