Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<normalization>
os.environ['OMP_NUM_THREADS'] = '2'
pd.set_option('display.max_colwidth', -1)
preprocessing = sm.get_preprocessing(BACKBONE)
 augmentation = albu.Compose([albu.HorizontalFlip(p=0.5),
albu.VerticalFlip(p=0.5),
albu.ShiftScaleRotate(rotate_limit=30, shift_limit=0.1, p=0.5)
 ] )<create_dataframe>
train_df = pd.read_csv(".. /input/train.csv",usecols=['question_text','target'])
 test_df = pd.read_csv(".. /input/test.csv")
 train_df, val_df = train_test_split(train_df, test_size=0.001, random_state = 200 )
train_generator = DataGenerator(
 directory=train_images_dest_path,
dataframe=X_train,
target_df=train,
batch_size=BATCH_SIZE,
target_size=(HEIGHT, WIDTH),
n_channels=CHANNELS,
n_classes=N_CLASSES,
preprocessing=preprocessing,
augmentation=augmentation,
seed=seed)
 valid_generator = DataGenerator(
 directory=validation_images_dest_path,
dataframe=X_val,
target_df=train,
batch_size=BATCH_SIZE,
target_size=(HEIGHT, WIDTH),
n_channels=CHANNELS,
n_classes=N_CLASSES,
preprocessing=preprocessing,
seed=seed )<choose_model_class>
train_df['target'].value_counts()
model = sm.Unet(backbone_name=BACKBONE,
encoder_weights='imagenet',
classes=N_CLASSES,
activation='sigmoid',
input_shape=(HEIGHT, WIDTH, CHANNELS))
checkpoint = ModelCheckpoint(model_path, monitor='val_loss', mode='min', save_best_only=True, save_weights_only=True)
 es = EarlyStopping(monitor='val_loss', mode='min', patience=ES_PATIENCE, restore_best_weights=True, verbose=1)
 rlrop = ReduceLROnPlateau(monitor='val_loss', mode='min', patience=RLROP_PATIENCE, factor=DECAY_DROP, min_lr=1e-6, verbose=1)
 metric_list = [dice_coef, sm.metrics.iou_score]
callback_list = [checkpoint, es, rlrop]
optimizer = RAdam(learning_rate=LEARNING_RATE, warmup_proportion=0.1)
 model.compile(optimizer=optimizer, loss=sm.losses.bce_dice_loss, metrics=metric_list)
 model.summary()<train_model>
val_df['target'].value_counts(normalize = True )
STEP_SIZE_TRAIN = len(X_train)//BATCH_SIZE
STEP_SIZE_VALID = len(X_val)//BATCH_SIZE
history = model.fit_generator(generator=train_generator,
steps_per_epoch=STEP_SIZE_TRAIN,
validation_data=valid_generator,
validation_steps=STEP_SIZE_VALID,
callbacks=callback_list,
epochs=EPOCHS,
verbose=2 ).history<load_pretrained>
model = load_model('.. /input/cloud-seg-resnet18-trainedlonger/resnet18_trained_longer.h5', custom_objects={'RAdam':RAdam, 'binary_crossentropy_plus_dice_loss':sm.losses.bce_dice_loss, 'dice_coef':dice_coef, 'iou_score':sm.metrics.iou_score, 'f1-score':sm.metrics.f1_score} )<define_search_space>
X_train = train_df['transform'].values
X_val = val_df['transform'].values
y_train = train_df['target'].values
y_val = val_df['target'].values
X_test = test_df['transform'].values
class_names = ['Fish ', 'Flower', 'Gravel', 'Sugar ']
best_tresholds = [.5,.5,.5,.35]
best_masks = [25000, 20000, 22500, 15000]
for index, name in enumerate(class_names):
print('%s treshold=%.2f mask size=%d' %(name, best_tresholds[index], best_masks[index]))<compute_train_metric>
%%time
maxlen = 50
tokenizer = text.Tokenizer(filters='\t
')
 tokenizer.fit_on_texts(list(X_train))
',oov_token=set(list(oov[0].values)))
 X_train = tokenizer.texts_to_sequences(X_train)
 X_val = tokenizer.texts_to_sequences(X_val)
 X_test = tokenizer.texts_to_sequences(X_test)
 x_train = sequence.pad_sequences(X_train, maxlen=maxlen, padding='post',truncating='post')
 x_val = sequence.pad_sequences(X_val, maxlen=maxlen, padding='post', truncating='post')
 x_test = sequence.pad_sequences(X_test, maxlen=maxlen, padding='post', truncating='post' )
train_metrics = get_metrics(model, train, X_train, train_images_dest_path, best_tresholds, best_masks, seed=seed, preprocessing=preprocessing, set_name='Train')
 display(train_metrics)
 validation_metrics = get_metrics(model, train, X_val, validation_images_dest_path, best_tresholds, best_masks, seed=seed, preprocessing=preprocessing, set_name='Validation')
 display(validation_metrics )<normalization>
word_index = tokenizer.word_index
embedding_matrix1 = np.zeros(( max(list(word_index.values())) + 1, 300), dtype = 'float32')
 embedding_matrix2 = np.zeros(( max(list(word_index.values())) + 1, 300), dtype = 'float32')
 len(word_index )
model = tta_segmentation(model, h_flip=True, v_flip=True, h_shift=(-10, 10), v_shift=(-10, 10), merge='mean' )<define_variables>
f = open('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt')
 for line in tqdm(f):
values = line.split(" ")
 if(len(values)< 200):
print("a")
 word = values[0]
if word not in word_index:
continue
embedding_matrix1[word_index[word]] = np.asarray(values[1:], dtype='float32')
 f.close()
test_df = []
for i in range(0, test.shape[0], 300):
batch_idx = list(range(i, min(test.shape[0], i + 300)))
 batch_set = test[batch_idx[0]: batch_idx[-1]+1]
test_generator = DataGenerator(
 directory=test_images_dest_path,
dataframe=batch_set,
target_df=submission,
batch_size=1,
target_size=(HEIGHT, WIDTH),
n_channels=CHANNELS,
n_classes=N_CLASSES,
preprocessing=preprocessing,
seed=seed,
mode='predict',
shuffle=False)
 preds = model.predict_generator(test_generator)
 for index, b in enumerate(batch_idx):
filename = test['image'].iloc[b]
image_df = submission[submission['image'] == filename].copy()
pred_masks = preds[index, ].round().astype(int)
 pred_rles = build_rles(pred_masks, reshape=(350, 525))
image_df['EncodedPixels'] = pred_rles
pred_masks_post = preds[index, ].astype('float32')
 for class_index in range(N_CLASSES):
pred_mask = pred_masks_post[...,class_index]
pred_mask = post_process(pred_mask, threshold=best_tresholds[class_index], min_size=best_masks[class_index])
 pred_masks_post[...,class_index] = pred_mask
pred_rles_post = build_rles(pred_masks_post, reshape=(350, 525))
image_df['EncodedPixels_post'] = pred_rles_post
test_df.append(image_df)
 sub_df = pd.concat(test_df )<define_variables>
f = open('.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt', encoding="utf8", errors='ignore')
 for line in tqdm(f):
values = line.split(" ")
 word = values[0]
if word in word_index:
embedding_matrix2[word_index[word]] = np.asarray(values[1:], dtype='float32')
 f.close()
images_to_inspect = np.random.choice(X_val['image'].unique() , 3, replace=False)
 inspect_set = train[train['image'].isin(images_to_inspect)].copy()
inspect_set_temp = []
inspect_generator = DataGenerator(
 directory=validation_images_dest_path,
dataframe=inspect_set,
target_df=train,
batch_size=1,
target_size=(HEIGHT, WIDTH),
n_channels=CHANNELS,
n_classes=N_CLASSES,
preprocessing=preprocessing,
seed=seed,
mode='fit',
shuffle=False)
 preds = model.predict_generator(inspect_generator)
 for index, b in enumerate(range(len(preds))):
filename = inspect_set['image'].iloc[b]
image_df = inspect_set[inspect_set['image'] == filename].copy()
pred_masks = preds[index, ].round().astype(int)
 pred_rles = build_rles(pred_masks, reshape=(350, 525))
image_df['EncodedPixels_pred'] = pred_rles
pred_masks_post = preds[index, ].astype('float32')
 for class_index in range(N_CLASSES):
pred_mask = pred_masks_post[...,class_index]
pred_mask = post_process(pred_mask, threshold=best_tresholds[class_index], min_size=best_masks[class_index])
 pred_masks_post[...,class_index] = pred_mask
pred_rles_post = build_rles(pred_masks_post, reshape=(350, 525))
image_df['EncodedPixels_pred_post'] = pred_rles_post
inspect_set_temp.append(image_df)
 inspect_set = pd.concat(inspect_set_temp)
 inspect_predictions(inspect_set, images_to_inspect, validation_images_dest_path, pred_col='EncodedPixels_pred' )<compute_train_metric>
tmp = pd.DataFrame(list(tokenizer.word_index.items()))
tmp[2] =(embedding_matrix1.sum(axis=1)==0)[1:]
tmp[3] =(embedding_matrix2.sum(axis=1)==0)[1:]
tmp = tmp[tmp[2] | tmp[3]][:4000]
name = tmp[0].values
indexes = tmp[1].values
embedding_matrix1[indexes] = 0
embedding_matrix2[indexes] = 0
inspect_predictions(inspect_set, images_to_inspect, validation_images_dest_path, pred_col='EncodedPixels_pred_post' )<predict_on_test>
%%time
token_name = {}
cnt = 1
for i in list(tokenizer.word_index.keys()):
if i in name:
token_name[i] = cnt
cnt += 1
else:
token_name[i] = 0
tokenizer.word_index = token_name
images_to_inspect_test = np.random.choice(sub_df['image'].unique() , 4, replace=False)
 inspect_predictions(sub_df, images_to_inspect_test, test_images_dest_path )<compute_train_metric>
X_train = train_df['transform'].values
X_val = val_df['transform'].values
y_train = train_df['target'].values
y_val = val_df['target'].values
X_test = test_df['transform'].values
X_train = tokenizer.texts_to_sequences(X_train)
 X_val = tokenizer.texts_to_sequences(X_val)
 X_test = tokenizer.texts_to_sequences(X_test)
 x_train_name = sequence.pad_sequences(X_train, maxlen=maxlen, padding='post',truncating='post')
 x_val_name = sequence.pad_sequences(X_val, maxlen=maxlen, padding='post', truncating='post')
 x_test_name = sequence.pad_sequences(X_test, maxlen=maxlen, padding='post', truncating='post' )
inspect_predictions(sub_df, images_to_inspect_test, test_images_dest_path, label_col='EncodedPixels_post' )<save_to_csv>
train_df['token'] = list(x_train)
 train_df['token_name'] = list(x_train_name )
submission_df = sub_df[['Image_Label' ,'EncodedPixels']]
submission_df.to_csv('submission.csv', index=False)
 display(submission_df.head() )<save_to_csv>
mask_zeros = np.ones(( name.shape[0] + 1, 600))
mask_zeros[0] = 0
submission_df_post = sub_df[['Image_Label' ,'EncodedPixels_post']]
submission_df_post.columns = ['Image_Label' ,'EncodedPixels']
submission_df_post.to_csv('submission_post.csv', index=False)
 display(submission_df_post.head() )<load_pretrained>
BatchNormalization, GlobalMaxPooling1D, GlobalAveragePooling1D, concatenate, Conv1D, Multiply, Add
if os.path.exists(train_images_dest_path):
shutil.rmtree(train_images_dest_path)
 if os.path.exists(validation_images_dest_path):
shutil.rmtree(validation_images_dest_path)
 if os.path.exists(test_images_dest_path):
shutil.rmtree(test_images_dest_path )<set_options>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
seed(10)
 set_random_seed(10)
 %matplotlib inline<install_modules>
class GlobalMinPooling1D(Layer):
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
return K.min(x, axis=1)
 def compute_output_shape(self, input_shape):
return(input_shape[0], input_shape[2])
!pip install keras-rectified-adam<define_variables>
class GlobalSumPooling1D(Layer):
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
return K.sum(x, axis=1)
 def compute_output_shape(self, input_shape):
return(input_shape[0], input_shape[2])
test_imgs_folder = '.. /input/understanding_cloud_organization/test_images/'
train_imgs_folder = '.. /input/understanding_cloud_organization/train_images/'
num_cores = multiprocessing.cpu_count()<load_from_csv>
def build_model(input_layer, input_layer_name, embedding_matrix):
x1 = Embedding(embedding_matrix.shape[0], 600, weights=[embedding_matrix], trainable= False )(input_layer)
 x2 = Embedding(name.shape[0] + 1, 600, trainable= True )(input_layer_name)
 x3 = Embedding(name.shape[0] + 1, 600, weights=[mask_zeros], trainable= False )(input_layer_name)
 x = Multiply()([x2, x3])
 x = Add()([x1, x])
 x = Bidirectional(CuDNNLSTM(128, kernel_initializer=he_uniform(seed=0), return_sequences=True))(x)
 x = SpatialDropout1D(0.2 )(x)
 y = Bidirectional(CuDNNGRU(128,kernel_initializer=he_uniform(seed=0), return_sequences=True))(x)
 a = GlobalAveragePooling1D()(y)
 b = GlobalMaxPooling1D()(y)
 c = GlobalMinPooling1D()(y)
 x = concatenate([a, b])
 x = Dense(32, activation="relu",kernel_initializer=he_uniform(seed=0))(x)
 x = Dense(1, activation="sigmoid",kernel_initializer=he_uniform(seed=0))(x)
 return x
train_df = pd.read_csv('.. /input/understanding_cloud_organization/train.csv')
 train_df.head()<feature_engineering>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
train_df = train_df[~train_df['EncodedPixels'].isnull() ]
train_df['Image'] = train_df['Image_Label'].map(lambda x: x.split('_')[0])
 train_df['Class'] = train_df['Image_Label'].map(lambda x: x.split('_')[1])
 classes = train_df['Class'].unique()
train_df = train_df.groupby('Image')['Class'].agg(set ).reset_index()
for class_name in classes:
train_df[class_name] = train_df['Class'].map(lambda x: 1 if class_name in x else 0)
 train_df.head()<define_variables>
train_df.reset_index(drop = True, inplace = True)
 val_df.reset_index(drop= True, inplace = True )
img_2_ohe_vector = {img:vec for img, vec in zip(train_df['Image'], train_df.iloc[:, 2:].values)}<split>
positive = train_df[train_df['target'] == 1][['token','token_name','target']]
negative = train_df[train_df['target'] == 0][['token','token_name','target']]
train_imgs, val_imgs = train_test_split(train_df['Image'].values,
test_size=0.2,
stratify=train_df['Class'].map(lambda x: str(sorted(list(x)))) ,
random_state=2019 )<define_variables>
for C in range(4):
if(C == 0):
embedding_matrix = np.concatenate([embedding_matrix1, embedding_matrix2], axis=1)
 elif(C == 1):
embedding_matrix = np.concatenate([embedding_matrix1, embedding_matrix2], axis=1)
 elif(C == 2):
embedding_matrix = np.concatenate([embedding_matrix2, embedding_matrix1], axis=1)
 else:
embedding_matrix = np.concatenate([ embedding_matrix2, embedding_matrix1], axis=1)
 neg1, neg2 = train_test_split(negative, test_size = 0.5, random_state = C*100)
 df1, df2 = pd.concat([neg1,positive], ignore_index=True), pd.concat([neg2,positive], ignore_index=True)
 input_layer = Input(( 50,), name="i1")
 input_layer_name = Input(( 50,), name = "i2")
 output_layer = build_model(input_layer, input_layer_name, embedding_matrix)
 model = Model([input_layer, input_layer_name], output_layer)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 model.fit({"i1": np.stack(df1['token'].values), "i2": np.stack(df1['token_name'].values)},
df1['target'], batch_size=128, verbose=2,shuffle=True,\
epochs=1, validation_data=({"i1": x_val, "i2": x_val_name}, val_df['target']))
model.fit({"i1":x_train, "i2": x_train_name}, train_df['target'], batch_size=256, verbose=2,shuffle=True,\
epochs=1, validation_data=({"i1": x_val, "i2": x_val_name}, val_df['target']))
model.layers[2].trainable = False
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 model.fit({"i1":x_train, "i2": x_train_name}, train_df['target'], batch_size=512, verbose=2,shuffle=True,\
epochs=1, validation_data=({"i1": x_val, "i2": x_val_name}, val_df['target']))
test_df[C] = model.predict({"i1": x_test, "i2": x_test_name}, batch_size=1024 ).flatten()
class DataGenenerator(Sequence):
def __init__(self, images_list=None, folder_imgs=train_imgs_folder,
batch_size=32, shuffle=True, augmentation=None,
resized_height=260, resized_width=260, num_channels=3):
self.batch_size = batch_size
self.shuffle = shuffle
self.augmentation = augmentation
if images_list is None:
self.images_list = os.listdir(folder_imgs)
 else:
self.images_list = deepcopy(images_list)
 self.folder_imgs = folder_imgs
self.len = len(self.images_list)// self.batch_size
self.resized_height = resized_height
self.resized_width = resized_width
self.num_channels = num_channels
self.num_classes = 4
self.is_test = not 'train' in folder_imgs
if not shuffle and not self.is_test:
self.labels = [img_2_ohe_vector[img] for img in self.images_list[:self.len*self.batch_size]]
def __len__(self):
return self.len
def on_epoch_start(self):
if self.shuffle:
random.shuffle(self.images_list)
 def __getitem__(self, idx):
current_batch = self.images_list[idx * self.batch_size:(idx + 1)* self.batch_size]
X = np.empty(( self.batch_size, self.resized_height, self.resized_width, self.num_channels))
y = np.empty(( self.batch_size, self.num_classes))
for i, image_name in enumerate(current_batch):
path = os.path.join(self.folder_imgs, image_name)
 img = cv2.resize(cv2.imread(path),(self.resized_height, self.resized_width)).astype(np.float32)
 if not self.augmentation is None:
augmented = self.augmentation(image=img)
 img = augmented['image']
X[i, :, :, :] = img/255.0
if not self.is_test:
y[i, :] = img_2_ohe_vector[image_name]
return X, y
def get_labels(self):
if self.shuffle:
images_current = self.images_list[:self.len*self.batch_size]
labels = [img_2_ohe_vector[img] for img in images_current]
else:
labels = self.labels
return np.array(labels )<concatenate>
test_df['preds'] =(test_df[0] + test_df[1] + test_df[2] + test_df[3])/4
<choose_model_class><EOS> 
y_te =(test_df['preds'] > 0.35 ).astype(np.int)
 submit_df = pd.DataFrame({"qid": test_df["qid"], "prediction": y_te})
 submit_df.to_csv("submission.csv", index=False )