Kaggle competition: Quora Insincere Questions Classification
%%time
bagged_catboost = ensemble_model(catboost, method='Bagging', n_estimators=10, optimize='RMSE' )<choose_model_class>
dim = 300
num_words = 50000
max_len = 100
print('Fiting tokenizer')
 tokenizer = Tokenizer(num_words=num_words)
 tokenizer.fit_on_texts(df['question_text'])
 print('text to sequence')
 x_train = tokenizer.texts_to_sequences(df['question_text'])
 print('pad sequence')
 x_train = pad_sequences(x_train,maxlen=max_len)
 y_train = df['target'].values
print(x_train.shape)
 print(y_train.shape )
%%time
bagged_xgboost = ensemble_model(xgboost, method='Bagging', n_estimators=10, optimize='RMSE' )<choose_model_class>
print('Glove...')
 def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_glov = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_glov[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()
%%time
blender = blend_models(optimize='RMSE', estimator_list=[
bayesian_ridge,
lightgbm,
catboost,
xgboost,
bagged_lightgbm,
bagged_catboost,
bagged_xgboost,
bagged_dt,
] )<choose_model_class>
print('Para...')
 EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_para = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_para[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()
%%time
best = automl(optimize='RMSE')
 best<find_best_params>
print('Wiki...')
 EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_wiki = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_wiki[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_wiki.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()
%%time
evaluate_model(best )<predict_on_test>
matrixes = [embedding_matrix_glov,embedding_matrix_wiki,embedding_matrix_para]
matrix = np.mean(matrixes,axis=0)
 del embedding_matrix_glov,embedding_matrix_wiki,embedding_matrix_para
gc.collect()
%%time
pred_holdouts = predict_model(best)
 pred_holdouts.head()<load_from_csv>
print('Loading test data...')
 df_final = pd.read_csv('.. /input/test.csv')
 df_final["question_text"].fillna("_
x_test=tokenizer.texts_to_sequences(df_final['question_text'])
 x_test = pad_sequences(x_test,maxlen=max_len)
 print('Test data loaded:',x_test.shape )
%%time
new_data = pd.read_csv('.. /input/tabular-playground-series-feb-2021/test.csv', index_col='id')
 predict_new = predict_model(best, data=new_data)
 predict_new.head()<save_to_csv>
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
 class Metrics(Callback):
def on_train_begin(self, logs={}):
self.val_f1s = []
def _threshold_search(self,y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
def on_epoch_end(self, epoch, logs={}):
y_val = self.validation_data[1]
y_pred = self.model.predict(self.validation_data[0])
 with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 search_result = self._threshold_search(y_val, y_pred)
 _val_f1 = search_result['f1']
self.val_f1s.append(_val_f1)
 print('— val_f1_on_epoch_end: %f '%(_val_f1))
return
def get_model(adam, trainable=False):
inp1 = Input(shape=(max_len,))
emb = Embedding(num_words, dim, weights=[matrix],trainable = trainable, )(inp1)
 filters = []
for f in [3,4,5]:
conv = Conv1D(64,f, activation='elu' )(emb)
 pool = GlobalMaxPooling1D()(conv)
 filters.append(pool)
 x = concatenate(filters)
 x = Dense(max_len, activation="relu" )(x)
 x = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=inp1, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy'])
 print(model.summary())
 return model
def get_opt() :
adam = optimizers.Adam()
print('LR:',K.eval(adam.lr))
return adam
def get_batch_size(opt):
batch_size = int(x_train.shape[0]*K.eval(opt.lr))
print('Batch size = ',batch_size)
 return batch_size
patience = 2
best_model=None
all_results = {}
train_meta = np.zeros(y_train.shape)
 test_meta = np.zeros(x_test.shape[0])
 splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=14 ).split(x_train, y_train))
for idx,(train_idx, valid_idx)in enumerate(splits):
print('----'+str(idx)+'-----')
 X_train1 = x_train[train_idx]
y_train1 = y_train[train_idx]
X_val = x_train[valid_idx]
y_val = y_train[valid_idx]
model_file = 'model_'+str(idx)+'.h5'
modelcheck = ModelCheckpoint(model_file,save_best_only=True)
 stop = EarlyStopping(patience=patience)
 reduce = CyclicLR(base_lr=0.001,
max_lr=0.004,
step_size=300.,
mode='exp_range',
gamma=0.99994)
 metrics = Metrics()
adam = get_opt()
batch_size = get_batch_size(adam)
 model = get_model(adam)
 history = model.fit(X_train1,y_train1,
batch_size=batch_size,
validation_data=(X_val,y_val),
epochs=100,
callbacks=[modelcheck,stop,reduce,metrics],
verbose=2)
 model.load_weights(model_file)
 print('training finished...')
 y_pred = model.predict(X_val,batch_size=batch_size, verbose=1)
 train_meta[valid_idx] = y_pred.reshape(-1)
 search_result = threshold_search(y_val, y_pred)
 print(search_result)
 y_pred = y_pred>search_result['threshold']
y_pred = y_pred.astype(int)
 print('RESULTS ON VALIDATION SET:
',classification_report(y_val,y_pred))
all_results[model_file] = search_result['f1']
y_pred = model.predict(x_test,batch_size=batch_size, verbose=1)
 test_meta += y_pred.reshape(-1)/ len(splits)
 if best_model is None or best_model['f1'] < search_result['f1']:
best_model={'model':model_file,'f1':search_result['f1']}
print('-'*80)
 print(all_results)
 print('-'*80)
 print(best_model)
 print('-'*80)
 train_meta = train_meta[train_meta > -1]
y_meta = train_meta[train_meta > -1]
<save_model><EOS> 
search_result = threshold_search(y_train, train_meta)
 print(search_result)
 df_subm = pd.DataFrame()
df_subm['qid'] = df_final.qid
df_subm['prediction'] = test_meta > search_result['threshold']
print(df_subm.head())
 df_subm.to_csv('submission.csv', index=False )