Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<load_from_csv>
tqdm.pandas()
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<feature_engineering>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print("Train shape : ", train.shape)
 print("Test shape : ", test.shape )
train.loc[train["var3"]==-999999,"var3"] = 2
all_clomuns = list(train.columns.values)
 all_clomuns.remove("ID")
 all_clomuns.remove("TARGET")
 f_values, p_values = f_classif(train[all_clomuns], train["TARGET"])
 p_values = -np.log10(p_values)
 p_values_dict = {}
for idx,value in enumerate(p_values):
if value not in p_values_dict:
p_values_dict[value] = [all_clomuns[idx]]
else:
p_values_dict[value].append(all_clomuns[idx])
 identical_pvalues = []
for value in p_values_dict.keys() :
if(len(p_values_dict[value])>1):
for name in p_values_dict[value][1:]:
all_clomuns.remove(name )<feature_engineering>
train["question_text"] = train["question_text"].str.lower()
test["question_text"] = test["question_text"].str.lower()
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
train["question_text"] = train["question_text"].apply(lambda x: clean_text(x))
test["question_text"] = test["question_text"].apply(lambda x: clean_text(x))
train2 = train[all_clomuns]
corrMatrix=train2.corr()
corrMatrix.loc[:,:] = np.tril(corrMatrix, k=-1)
 already_in = set()
result = []
for col in corrMatrix:
perfect_corr = corrMatrix[col][corrMatrix[col] > 0.9].index.tolist()
if perfect_corr and col not in already_in:
already_in.update(set(perfect_corr))
perfect_corr.append(col)
 result.append(perfect_corr)
 for pair in result:
for name in pair[1:]:
if name in all_clomuns:
all_clomuns.remove(name )<train_model>
embed_size = 300
max_features = None
maxlen = 72
X = train["question_text"].fillna("_na_" ).values
X_test = test["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features, filters='')
 tokenizer.fit_on_texts(list(X))
X = tokenizer.texts_to_sequences(X)
 X_test = tokenizer.texts_to_sequences(X_test)
 X = pad_sequences(X, maxlen=maxlen)
 X_test = pad_sequences(X_test, maxlen=maxlen)
 Y = train['target'].values
sub = test[['qid']]
y_train = train['TARGET'].values
X_train = train[all_clomuns].values
id_test = test['ID']
X_test = test[all_clomuns].values
len_train = len(X_train)
 len_test = len(X_test)
 clf = xgb.XGBClassifier(missing=np.nan, max_depth=6, n_estimators=500, learning_rate=0.02, nthread=4, subsample=0.75, colsample_bytree=0.75, seed=1024)
 X_fit, X_eval, y_fit, y_eval= train_test_split(X_train, y_train, test_size=0.3)
 clf.fit(X_train, y_train, early_stopping_rounds=30, eval_metric="auc", eval_set=[(X_eval, y_eval)],verbose=1)
 y_pred= clf.predict_proba(X_test)[:,1]
submission = pd.DataFrame({"ID":id_test, "TARGET":y_pred})
 submission.to_csv("submission6.csv", index=False )<load_from_csv>
del train, test
gc.collect()
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 ignored_columns = ['ID', 'TARGET']
C = df_train.columns
eps = 1e-10
dropped_columns = set()
print('Identifing low-variance columns...', end=' ')
 for c in C:
if df_train[c].var() < eps:
dropped_columns.add(c)
 print('done!')
 C = list(set(C)- dropped_columns - set(ignored_columns))
print('Identifying duplicate columns...', end=' ')
 for i, c1 in enumerate(C):
f1 = df_train[c1].values
for j, c2 in enumerate(C[i+1:]):
f2 = df_train[c2].values
if np.all(f1 == f2):
dropped_columns.add(c2)
 print('done!')
 C = list(set(C)- dropped_columns - set(ignored_columns))
print('
print('
df_train.drop(dropped_columns, axis=1, inplace=True)
 df_test.drop(dropped_columns, axis=1, inplace=True )<split>
word_index = tokenizer.word_index
max_features = len(word_index)+1
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100 and o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100 and o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
y_learning = df_train['TARGET'].values
X_learning = df_train.drop(['ID','TARGET'], axis=1 ).values
id_test = df_test['ID']
X_test = df_test.drop(['ID'], axis=1 ).values
X_train_valid, X_local_test, y_train_valid, y_local_test = train_test_split(
 X_learning, y_learning, test_size=0.168, random_state=1 
)
 X_fit, X_eval, y_fit, y_eval= train_test_split(
 X_train_valid, y_train_valid, test_size=0.2, random_state=1 
)
 print('
print('
print('<train_on_grid>
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_3 = load_para(word_index)
 embedding_matrix = np.mean(( embedding_matrix_1, embedding_matrix_3), axis=0)
 del embedding_matrix_1, embedding_matrix_3
gc.collect()
np.shape(embedding_matrix )
clf = xgb.XGBClassifier(missing=np.nan, max_depth=5,
n_estimators=1000, learning_rate=0.01,
subsample=0.5, colsample_bytree=0.9, seed=4242)
 clf.fit(X_fit, y_fit, early_stopping_rounds=50, eval_metric="auc", eval_set=[(X_eval, y_eval)])
 auc_train = roc_auc_score(y_fit, clf.predict_proba(X_fit)[:,1])
 auc_valid = roc_auc_score(y_eval, clf.predict_proba(X_eval)[:,1])
 auc_test = roc_auc_score(y_local_test, clf.predict_proba(X_local_test)[:,1])
 print('
-----------------------')
 print(' AUC train: %.5f'%auc_train)
 print(' AUC valid: %.5f'%auc_valid)
 print(' AUC test : %.5f'%auc_test)
 print('-----------------------')
 print('
Model parameters...')
 print(clf.get_params())
 print('
-----------------------
')
 y_pred= clf.predict_proba(X_test)[:,1]
submission = pd.DataFrame({"ID":id_test, "TARGET":y_pred})
 submission.to_csv("submission.csv", index=False)
 print('Completed!' )<set_options>
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale
class Capsule(Layer):
def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(tf.keras.backend.batch_dot(c, u_hat_vecs, [2, 2]))
if i < self.routings - 1:
b = tf.keras.backend.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule )
%matplotlib inline
warnings.filterwarnings('ignore', category=UserWarning, module='bs4')
<load_from_csv>
def capsule() :
K.clear_session()
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(rate=0.2 )(x)
 x = Bidirectional(CuDNNGRU(100, return_sequences=True,
kernel_initializer=glorot_normal(seed=12300), recurrent_initializer=orthogonal(gain=1.0, seed=10000)) )(x)
 x = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True )(x)
 x = Flatten()(x)
 x = Dense(100, activation="relu", kernel_initializer=glorot_normal(seed=12300))(x)
 x = Dropout(0.12 )(x)
 x = BatchNormalization()(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam() ,)
 return model
train = pd.read_csv('/kaggle/input/sentiment-analysis-on-movie-reviews/train.tsv.zip', sep='\t')
 test = pd.read_csv('/kaggle/input/sentiment-analysis-on-movie-reviews/test.tsv.zip', sep='\t')
 train.shape, test.shape<choose_model_class>
def f1_smart(y_true, y_pred):
args = np.argsort(y_pred)
 tp = y_true.sum()
fs =(tp - np.cumsum(y_true[args[:-1]])) / np.arange(y_true.shape[0] + tp - 1, tp, -1)
 res_idx = np.argmax(fs)
 return 2 * fs[res_idx],(y_pred[args[res_idx]] + y_pred[args[res_idx + 1]])/ 2
lemmatizer = WordNetLemmatizer()<categorify>
kfold = StratifiedKFold(n_splits=5, random_state=10, shuffle=True)
 bestscore = []
y_test = np.zeros(( X_test.shape[0],))
for i,(train_index, valid_index)in enumerate(kfold.split(X, Y)) :
X_train, X_val, Y_train, Y_val = X[train_index], X[valid_index], Y[train_index], Y[valid_index]
filepath="weights_best.h5"
checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=2, save_best_only=True, mode='min')
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.6, patience=1, min_lr=0.0001, verbose=2)
 earlystopping = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=2, verbose=2, mode='auto')
 callbacks = [checkpoint, reduce_lr]
model = capsule()
if i == 0:print(model.summary())
 model.fit(X_train, Y_train, batch_size=512, epochs=6, validation_data=(X_val, Y_val), verbose=2, callbacks=callbacks, 
)
 model.load_weights(filepath)
 y_pred = model.predict([X_val], batch_size=1024, verbose=2)
 y_test += np.squeeze(model.predict([X_test], batch_size=1024, verbose=2)) /5
f1, threshold = f1_smart(np.squeeze(Y_val), np.squeeze(y_pred))
print('Optimal F1: {:.4f} at threshold: {:.4f}'.format(f1, threshold))
bestscore.append(threshold )
<import_modules><EOS> 
y_test = y_test.reshape(( -1, 1))
pred_test_y =(y_test>np.mean(bestscore)).astype(int)
 sub['prediction'] = pred_test_y
sub.to_csv("submission.csv", index=False )