Kaggle competition: Computational Intelligence Project
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim 
<compute_test_metric>
random_seed = 1213
np.random.seed(random_seed)
def f1(y_true, y_pred):
def recall(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true*y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives/(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true*y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives/(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))<choose_model_class>
train = pd.read_csv('.. /input/ciproject/persianMNIST_train.csv')
 test = pd.read_csv('.. /input/ciproject/persianMNIST_test.csv')
 Y = train['0']
X = train.drop(labels="0", axis=1)
 X = X.values.reshape(-1, 28, 28, 1)
 test = test.values.reshape(-1, 28, 28, 1)
 X_train, X_val, Y_train, Y_val = train_test_split(X, Y, test_size=0.10, random_state=random_seed)
 print(X.shape, test.shape )
def model_lstm_att(embedding_matrix):
inp = Input(shape=(MAXLEN,))
x = Embedding(MAX_FEATURES, EMBED_SIZE, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(32, return_sequences=True))(x)
 att = Attention(MAXLEN )(x)
 y = Dense(32, activation='relu' )(att)
 y = Dropout(0.1 )(y)
 outp = Dense(1, activation='sigmoid' )(y)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[f1])
 return model<find_best_params>
learning_rate_reduction = ReduceLROnPlateau(monitor = 'val_acc',
patience = 3,
verbose = 1,
factor = 0.5,
min_lr = 0.0001)
 es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=15)
 def new_model(hidden=512, learning_rate=0.00128):
INPUT = Input(( 28, 28, 1))
inputs = Conv2D(64,(7, 7), activation='relu', padding='same' )(INPUT)
 inputs = MaxPool2D(pool_size=(5,5), strides=(2,2))(inputs)
 inputs = BatchNormalization()(inputs)
 inputs = Activation('relu' )(inputs)
 inputs = Dropout(0.5 )(inputs)
 tower_1 = Conv2D(64,(1, 1), activation='relu', padding='same' )(inputs)
 tower_1 = Conv2D(64,(2, 2), activation='relu', padding='same' )(tower_1)
 tower_1 = Conv2D(64,(3, 3), activation='relu', padding='same' )(tower_1)
 tower_1 = MaxPool2D(pool_size=(3,3), strides=(2,2))(tower_1)
 tower_1 = BatchNormalization()(tower_1)
 tower_2 = Conv2D(64,(2, 2), activation='relu', padding='same' )(inputs)
 tower_2 = Conv2D(64,(3, 3), activation='relu', padding='same' )(tower_2)
 tower_2 = Conv2D(64,(5, 5), activation='relu', padding='same' )(tower_2)
 tower_2 = MaxPool2D(pool_size=(3,3), strides=(2,2))(tower_2)
 tower_2 = BatchNormalization()(tower_2)
 tower_3 = Conv2D(64,(3, 3), activation='relu', padding='same' )(inputs)
 tower_3 = Conv2D(64,(3, 3), activation='relu', padding='same' )(tower_3)
 tower_3 = Conv2D(64,(5, 5), activation='relu', padding='same' )(tower_3)
 tower_3 = MaxPool2D(pool_size=(3,3), strides=(2,2))(tower_3)
 tower_3 = BatchNormalization()(tower_3)
 x = Add()([tower_1, tower_2, tower_3])
 x = Activation('relu' )(x)
 x = Conv2D(128,(5, 5), activation='relu', padding='same' )(x)
 x = MaxPool2D(pool_size=(5,5), strides=(3,3))(x)
 x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 x = Flatten()(x)
 x = Dense(hidden, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 preds = Dense(10, activation='softmax', name='preds' )(x)
 model = Model(inputs=INPUT, outputs=preds)
 optimizer = Adam(lr=learning_rate)
 model.compile(optimizer=optimizer, loss='sparse_categorical_crossentropy', metrics=['acc'])
 return model
model = new_model()
def train_pred(model, epochs=2):
for e in range(epochs):
model.fit(xtrain, ytrain, batch_size=512, epochs=3, validation_data=(xval, yval))
pred_val_y = model.predict([xval], batch_size=1024, verbose=0)
 best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(yval,(pred_val_y > thresh ).astype(int))
if score > best_score:
best_thresh = thresh
best_score = score
print("Val F1 Score: {:.4f}".format(best_score))
pred_test_y = model.predict([xtest], batch_size=1024, verbose=0)
 return pred_val_y, pred_test_y, best_score<load_pretrained>
datagen = ImageDataGenerator(featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=5,
zoom_range=0.05,
shear_range=0.02,
width_shift_range=0.05,
height_shift_range=0.05,
horizontal_flip=False,
vertical_flip=False )
paragram = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
embedding_matrix_para = load_glove_matrix(tokenizer.word_index, load_embed(paragram))<compute_test_metric>
epochs = 200
batch_size = 128
print("Learning Properties: Epoch:%i \t Batch Size:%i" %(epochs, batch_size))
predict_accumulator = np.zeros(model.predict(test ).shape)
 accumulated_history = []
for i in range(1, 6):
X_train, X_val, Y_train, Y_val = train_test_split(X, Y, test_size=0.20, random_state=random_seed*i)
 model = new_model(100, 0.008/i)
 datagen.fit(X_train)
 history = model.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_size),
epochs=epochs, validation_data=(X_val, Y_val), verbose=3,
steps_per_epoch=X_train.shape[0]//batch_size,
callbacks=[learning_rate_reduction, es],
workers=4)
 loss, acc = model.evaluate(X, Y)
 if acc > 0.75:
predict_accumulator += model.predict(test)*acc
accumulated_history.append(history)
 print("Current Predictions on fold number %i", i)
 print(*np.argmax(predict_accumulator, axis=1), sep='\t' )
embedding_matrix = np.mean([embedding_matrix_glove, embedding_matrix_para], axis=0 )<predict_on_test>
print("Completed Training.")
 results = np.argmax(predict_accumulator, axis=1)
 results = pd.Series(results, name="result")
 print("Saving prediction to output...")
 submission = pd.concat([pd.Series(range(0, test.shape[0]), name="ids"), results], axis=1)
 submission.to_csv('prediction.csv', index=False )