Kaggle competition: Птица или самолет
<SOS> metric: 
categorizationaccuracy Kaggle data source: bird-or-aircraft<choose_model_class>
warnings.filterwarnings('ignore')
 np.seed = 1324
sns.set(style="white")
 %matplotlib inline
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<train_model>
import cv2
import sys
import os
import keras
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
 def f1(y_true, y_pred):

 def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))<define_search_model>
from keras.models import Sequential
from keras import layers
from keras.layers import Conv2D
from keras.layers import MaxPooling2D
from keras.layers import Flatten
from keras.layers import Dense
from keras.layers import SpatialDropout2D
from keras.layers import Dropout
from keras.layers import Activation, BatchNormalization
from keras.optimizers import SGD
from keras import regularizers
from keras.callbacks import TerminateOnNaN, ModelCheckpoint, LearningRateScheduler
from keras.utils.np_utils import to_categorical
import tensorflow as tf
from PIL import Image, ImageEnhance
from keras.utils import np_utils
from keras.preprocessing.image import ImageDataGenerator
def model_lstm_atten(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.1 )(x)
 x = Bidirectional(CuDNNLSTM(40, return_sequences=True))(x)
 y = Bidirectional(CuDNNGRU(40, return_sequences=True))(x)
 atten_1 = Attention(maxlen )(x)
 atten_2 = Attention(maxlen )(y)
 avg_pool = GlobalAveragePooling1D()(y)
 max_pool = GlobalMaxPooling1D()(y)
 conc = concatenate([atten_1, atten_2, avg_pool, max_pool])
 conc = Dense(16, activation="relu" )(conc)
 conc = Dropout(0.1 )(conc)
 outp = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[f1])
 return model<predict_on_test>
def submit_file(y_pred, filename):
submit = pd.Series(y_pred.reshape(y_pred.shape[0]), name='target' ).replace({0: 'Bird', 1: 'Airplane'})
 submit.to_csv(filename +'.csv', index_label='id', header=True)
 print('file created' )
def train_pred(model, train_X, train_y, val_X, val_y, epochs=2, callback=None):
for e in range(epochs):
model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y), callbacks = callback, verbose=0)
 pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)
 best_score = metrics.f1_score(val_y,(pred_val_y > 0.33 ).astype(int))
print("Epoch: ", e, "- Val F1 Score: {:.4f}".format(best_score))
pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)
 print('=' * 60)
 return pred_val_y, pred_test_y, best_score<split>
def flip_image(array, show=False):

 image_obj = Image.fromarray(array.astype('uint8'))
rotated_image = image_obj.transpose(Image.FLIP_LEFT_RIGHT)
 flipped_array = np.array(rotated_image, dtype=np.int)
 if show:
plt.imshow(flipped_array)
 return flipped_array
def contrast_image(array, c_level=4.0, show=False):

 image_obj = Image.fromarray(array.astype('uint8'))
enhancer = ImageEnhance.Contrast(image_obj)
 enhanced_im = enhancer.enhance(c_level)
 contrasted_array = np.array(enhanced_im, dtype=np.int)
 if show:
plt.imshow(contrasted_array)
 return contrasted_array
train_X, test_X, train_y, word_index = load_and_prec()
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_3 = load_para(word_index )<normalization>
train_x = pd.read_csv('.. /input/train_x.csv', index_col=0, header=None)
 train_y = pd.read_csv('.. /input/train_y.csv', index_col=0)
 test_x = pd.read_csv('.. /input/test_x.csv', index_col=0, header=None )
embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_3], axis = 0)
 np.shape(embedding_matrix )<compute_train_metric>
train_y.replace(['Bird', 'Airplane'], [0,1], inplace=True)
 X = train_x.values.reshape(7200, 32, 32, 3)
 X_test = test_x.values.reshape(4800, 32, 32, 3)
 y_train = train_y.values
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result<split>
viz_img(X[100], y_train[100] )
DATA_SPLIT_SEED = 2018
clr = CyclicLR(base_lr=0.001, max_lr=0.002,
step_size=300., mode='exp_range',
gamma=0.99994)
 train_meta = np.zeros(train_y.shape)
 test_meta = np.zeros(test_X.shape[0])
 splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=DATA_SPLIT_SEED ).split(train_X, train_y))
for idx,(train_idx, valid_idx)in enumerate(splits):
X_train = train_X[train_idx]
y_train = train_y[train_idx]
X_val = train_X[valid_idx]
y_val = train_y[valid_idx]
model = model_lstm_atten(embedding_matrix)
 pred_val_y, pred_test_y, best_score = train_pred(model, X_train, y_train, X_val, y_val, epochs = 8, callback = [clr,])
 train_meta[valid_idx] = pred_val_y.reshape(-1)
 test_meta += pred_test_y.reshape(-1)/ len(splits )<save_to_csv>
viz_img(X[452], y_train[452] )
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.prediction = test_meta > 0.33
sub.to_csv("submission.csv", index=False)
<compute_test_metric>
X = X.astype('float32')
 X_test = X_test.astype('float32')
 x_train, x_test, y_train, y_test = train_test_split(X, y_train, test_size=0.3, random_state=12000 )
f1_score(y_true=train_y, y_pred=train_meta > 0.33 )<set_options>
mean = np.mean(x_train, axis=(0,1,2,3))
std = np.std(x_train,axis=(0,1,2,3))
x_train =(x_train-mean)/(std + 1e-7)
 x_test =(x_test-mean)/(std + 1e-7)
 X_test =(X_test-mean)/(std + 1e-7 )
%matplotlib inline 
<load_from_csv>
weight_decay = 1e-4
model = Sequential()
model.add(Conv2D(32,(3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay),
input_shape=x_train.shape[1:]))
model.add(Activation('elu'))
model.add(BatchNormalization())
 model.add(Conv2D(32,(3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
 model.add(Activation('elu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.2))
model.add(Conv2D(64,(3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
 model.add(Activation('elu'))
model.add(BatchNormalization())
 model.add(Conv2D(64,(3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
 model.add(Activation('elu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.3))
model.add(Conv2D(128,(3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
 model.add(Activation('elu'))
model.add(BatchNormalization())
 model.add(Conv2D(128,(3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
 model.add(Activation('elu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.4))
model.add(Flatten())
 model.add(Dense(1, activation='sigmoid'))
model.summary()
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train.shape)
 print("Test shape : ",test.shape )<define_variables>
datagen = ImageDataGenerator(
 rotation_range=15,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True, 
)
 datagen.fit(x_train )
EMBED_SIZE = 300
MAX_FEATURES = 100000
MAXLEN = 40<feature_engineering>
batch_size = 64
opt_rms = keras.optimizers.rmsprop(lr=0.001,decay=1e-6)
 model.compile(loss='binary_crossentropy', optimizer=opt_rms, metrics=['accuracy'])
 model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),
steps_per_epoch=x_train.shape[0] // batch_size, epochs=125,
verbose=1,validation_data=(x_test,y_test),callbacks=[LearningRateScheduler(lr_schedule)] )
<concatenate><EOS> 
y_pred = model.predict_classes(X_test )