Kaggle competition: Quora Insincere Questions Classification
total_df.Cabin = total_df.Cabin.fillna('X')
 total_df.Cabin = total_df.Cabin.apply(lambda x: x[0])
 total_df.Cabin = total_df.Cabin == 'X'
total_df.Cabin = total_df.Cabin.apply(lambda x: 0 if x == False else 1)
 print(total_df.Cabin.value_counts() )<count_values>
train_df = pd.read_csv(".. /input/train.csv")
 X_train = train_df["question_text"].fillna("dieter" ).values
test_df = pd.read_csv(".. /input/test.csv")
 X_test = test_df["question_text"].fillna("dieter" ).values
y = train_df["target"]
total_df.Embarked = total_df.Embarked.fillna(total_df.Embarked.mode() [0])
 total_df.Embarked.value_counts()<drop_column>
tqdm.pandas()
total_df = total_df.drop(['Ticket'], axis = 1 )<categorify>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
temp = copy.deepcopy(total_df[total_df.columns[total_df.columns != 'Age']])
 temp.Title.replace(to_replace = ['Mr'] , value = 0 , inplace = True)
 temp.Title.replace(to_replace = ['Mrs'] , value = 1 , inplace = True)
 temp.Title.replace(to_replace = ['Miss'] , value = 2, inplace = True)
 temp.Title.replace(to_replace = ['respect'] , value = 3 , inplace = True)
 temp = temp.agg(LabelEncoder().fit_transform)
 temp['Age'] = total_df['Age']
Corr = temp.corr() ["Age"].apply(lambda x : abs(x)).sort_values(ascending = False)
 print(Corr[:10])
 total_df['Missing_Age'] = total_df.Age.isnull()<groupby>
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"that's" : "that is",
"there's" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not",
"tryin'":"trying"}
total_df.Age = temp.groupby(['Pclass','Title'])['Age'].transform(lambda v: v.fillna(v.median()))
checking_missing(total_df )<categorify>
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = str(x)
 x = re.sub(r'[0-9]{5,}', '
x = re.sub(r'[0-9]{4}', '
x = re.sub(r'[0-9]{3}', '
x = re.sub(r'[0-9]{2}', '
return x
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
text = mispellings_re.sub(replace, text)
 return text
LE_columns = ['FamSize','Fare_Group','Sex','Age_Group','Missing_Age']
OHE_columns = ['Embarked','Title']
Other_columns = total_df.columns[~total_df.columns.isin(LE_columns)& ~total_df.columns.isin(OHE_columns)]
LE_data = total_df[LE_columns]
OHE_data = total_df[OHE_columns]
Other_data = total_df[Other_columns]
for col in LE_data:
LE_data[col] = LabelEncoder().fit_transform(LE_data[col])
 OHE_data = pd.get_dummies(OHE_data)
 total_df = pd.concat([Other_data,LE_data,OHE_data], axis = 1)
 total_df.head(5 )<drop_column>
X_train["question_text"] = X_train["question_text"].map(clean_numbers)
 X_test["question_text"] = X_test["question_text"].map(clean_numbers)
 X_train["question_text"] = X_train["question_text"].map(replace_typical_misspell)
 X_test["question_text"] = X_test["question_text"].map(replace_typical_misspell)
train_df = train_df.drop(['PassengerId','Survived'],axis = 1)
 test_df = test_df.drop(['PassengerId','Survived'], axis = 1)
 ntrain = len(train_df)
 ntest = len(test_df )<define_search_space>
SEQ_LEN = 100
maxlen = 100
max_features = 50000
tokenizer = text.Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(X_train)+ list(X_test))
X_train = tokenizer.texts_to_sequences(X_train)
 X_test = tokenizer.texts_to_sequences(X_test)
 x_train = sequence.pad_sequences(X_train, maxlen=maxlen)
 x_test = sequence.pad_sequences(X_test, maxlen=maxlen )
xgbParams = {'max_depth':np.arange(3,11,2),'learning_rate':np.arange(0.01,0.2,0.05)
 ,'n_estimator':np.arange(100,1100,50),'gamma':np.arange(0.01,0.3,0.05)}
ABParams = {'n_estimators':np.arange(100,1100,50),'learning_rate':np.arange(0.01,0.2,0.05)}
RFParams = {'n_estimators':np.arange(100,1100,50),'max_depth':np.arange(3,11,2),
'min_samples_split':np.arange(2,20,2),'min_samples_leaf':np.arange(1,6,2)}
ETParams = {'n_estimators':np.arange(100,1100,50),'max_depth':np.arange(3,11,2),
'min_samples_leaf':np.arange(1,6,2),'verbose':[0]}
gbcParams = {'n_estimators' : np.arange(100,1100,50), 'learning_rate' : np.arange(0.01,0.2,0.05)
 , 'min_samples_split' : np.arange(2,20,2), 'min_samples_leaf' : np.arange(1,6,2),
'max_depth' : np.arange(3,11,2)}
LGBMCParams = {'n_estimators' : np.arange(100,1000,50), 'learning_rate' : np.arange(0.01,0.2,0.05),
'num_leaves' : np.arange(2,12,2)}
def tuneParams(classifier, params, train_x, train_y, cv = 5):
md = GridSearchCV(classifier, params, cv = cv, scoring = 'accuracy', n_jobs = -1)
 md.fit(train_x,train_y)
 return md.best_params_,np.round(md.best_score_*100,2)
 def tuneParamsRandom(classifier, params, train_x, train_y, cv=5):
rs = RandomizedSearchCV(classifier, params, n_iter = 30,scoring = 'accuracy', n_jobs = -1, verbose = 0)
 rs.fit(train_x,train_y)
 return rs.best_params_, abs(rs.best_score_ )<train_on_grid>
EMBEDDING_FILE_GLOVE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index_glove = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE_GLOVE))
all_embs_glove = np.stack(embeddings_index_glove.values())
 emb_mean_glove,emb_std_glove = -0.005838499,0.48782197
embed_size_glove = all_embs_glove.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_glove = np.random.normal(emb_mean_glove, emb_std_glove,(nb_words, embed_size_glove))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector_glove = embeddings_index_glove.get(word)
 if embedding_vector_glove is not None: embedding_matrix_glove[i] = embedding_vector_glove
xgb_best_Params,xgb_best_score = tuneParamsRandom(XGBClassifier() ,xgbParams,train_df,train_label)
 print("XGB:",xgb_best_Params,xgb_best_score)
 AB_best_Params,AB_best_score = tuneParamsRandom(AdaBoostClassifier() ,ABParams,train_df,train_label)
 print("AdaBoost:",AB_best_Params,AB_best_score)
 RF_best_Params,RF_best_score = tuneParamsRandom(RandomForestClassifier() ,RFParams,train_df,train_label)
 print("RandomForest:",RF_best_Params,RF_best_score)
 ET_best_Params,ET_best_score = tuneParamsRandom(ExtraTreesClassifier() ,ETParams,train_df,train_label)
 print("ExtraTrees:",ET_best_Params,ET_best_score)
 GBC_best_Params,GBC_best_score = tuneParamsRandom(GradientBoostingClassifier() ,gbcParams,train_df,train_label)
 print("GradientBoostingClassifier:",GBC_best_Params,GBC_best_score)
 LGBMC_best_Params, LGBMC_best_score = tuneParamsRandom(LGBMClassifier() ,LGBMCParams,train_df,train_label)
 print("LGBMClassifier:",LGBMC_best_Params,LGBMC_best_score )<split>
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale
class Capsule(Layer):
def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(tf.keras.backend.batch_dot(c, u_hat_vecs, [2, 2]))
if i < self.routings - 1:
b = tf.keras.backend.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule )
kf = KFold(n_splits = 5, random_state = 2019)
 def fold_training(model, train_x, train_y, test_x):
oof_train = np.zeros(( ntrain))
oof_test = np.zeros(( ntest))
oof_kf_test = np.zeros(( 5,ntest))
for i,(train_idx,test_idx)in enumerate(kf.split(train_x)) :
kf_train_x = train_x.iloc[train_idx]
kf_train_y = train_y.iloc[train_idx]
kf_test_x = train_x.iloc[test_idx]
model.fit(kf_train_x,kf_train_y)
 oof_train[test_idx] = model.predict(kf_test_x)
 oof_kf_test[i,:] = model.predict(test_x)
 oof_test = oof_kf_test.mean(axis=0)
 return oof_train.reshape(-1,1), oof_test.reshape(-1,1 )<train_model>
def capsule() :
K.clear_session()
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size_glove, weights=[embedding_matrix_glove], trainable=False )(inp)
 x = SpatialDropout1D(rate=0.2 )(x)
 x = Bidirectional(CuDNNLSTM(160, return_sequences=True,
kernel_initializer=glorot_normal(seed=12300), recurrent_initializer=orthogonal(gain=1.0, seed=10000)) )(x)
 x = Capsule(num_capsule=15, dim_capsule=12, routings=8, share_weights=True )(x)
 x = Flatten()(x)
 x = Dense(100, activation=None, kernel_initializer=glorot_normal(seed=12300))(x)
 x = Dropout(0.2 )(x)
 x = BatchNormalization()(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam() ,)
 return model
model_glove = capsule()
xgb_oof_train,xgb_oof_test = fold_training(XGBClassifier(**xgb_best_Params),train_df,train_label,test_df)
 AB_oof_train,AB_oof_test = fold_training(AdaBoostClassifier(**AB_best_Params),train_df,train_label,test_df)
 RF_oof_train,RF_oof_test = fold_training(RandomForestClassifier(**RF_best_Params),train_df,train_label,test_df)
 ET_oof_train,ET_oof_test = fold_training(ExtraTreesClassifier(**ET_best_Params),train_df,train_label,test_df)
 gbc_oof_train,gbc_oof_test = fold_training(GradientBoostingClassifier(**GBC_best_Params),train_df,train_label,test_df)
 LGBMC_oof_train,LGBMC_oof_test = fold_training(LGBMClassifier(**LGBMC_best_Params),train_df,train_label,test_df )<create_dataframe>
batch_size = 128
epochs = 3
final_train_df = pd.DataFrame(np.concatenate(( xgb_oof_train,AB_oof_train,
RF_oof_train,ET_oof_train,gbc_oof_train,LGBMC_oof_train),axis=1))
final_test_df = pd.DataFrame(np.concatenate(( xgb_oof_test,AB_oof_test,
RF_oof_test,ET_oof_test,gbc_oof_test,LGBMC_oof_test),axis=1))
LRParams = {'penalty':['l1','l2'],'C':[0.01,0.05,0.1,0.2]}
LR_best_Params, LR_best_score = tuneParams(LogisticRegression() ,LRParams,final_train_df, train_label)
 print("LR:",LR_best_Params,LR_best_score )<train_model>
X_tra, X_val, y_tra, y_val = train_test_split(x_train, y, test_size = 0.1, random_state=42 )
LR = LogisticRegression(**LR_best_Params)
 LR.fit(final_train_df,train_label)
 prediction = LR.predict(final_test_df )<save_to_csv>
hist_glove = model_glove.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),
verbose=True)
 pred_glove_y = model_glove.predict(x_test, batch_size=1024)
 pred_glove_y =(pred_glove_y[:,0] > 0.33 ).astype(np.int )
submission = pd.DataFrame({'PassengerId':test_Ids,'Survived':prediction})
 submission.to_csv('submission_v9.csv',index = False )<set_options>
submit_df = pd.DataFrame({"qid": test_df["qid"], "prediction": pred_glove_y})
 submit_df.to_csv("submission.csv", index=False )