Kaggle competition: Quora Insincere Questions Classification
all_data['new_item'] =(all_data['date_block_num']==all_data['item_first_month'] ).astype(int)
 all_data['new_item'].value_counts()<categorify>
from sklearn.model_selection import GridSearchCV,StratifiedKFold
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import cross_val_score,train_test_split
from scipy import stats
from sklearn import metrics
from keras.models import Sequential
from keras.layers import Dense
from keras.preprocessing.text import Tokenizer
import string,re
from collections import Counter
import nltk
from nltk.corpus import stopwords
import spacy
from spacy.lang.en.stop_words import STOP_WORDS
from spacy.lang.en import English
all_data = downcast_dtypes(all_data)
 del grid, gb
gc.collect() ;
all_data.head()<merge>
import time
from tqdm import tqdm
import math
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D,CuDNNLSTM
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers
from sklearn import model_selection as ms
from sklearn.model_selection import KFold,StratifiedKFold
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.svm import LinearSVC
from keras.layers import Input, Dense, Embedding, concatenate
from keras.layers import CuDNNGRU, Bidirectional, GlobalAveragePooling1D, GlobalMaxPooling1D
from keras.preprocessing import text, sequence
from keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.layers import *
from keras.callbacks import *
cols_to_rename = list(all_data.columns.difference(index_cols+['item_first_month']))
shift_range = [1, 2, 3, 4, 5, 12]
for month_shift in tqdm_notebook(shift_range):
train_shift = all_data[index_cols + cols_to_rename].copy()
train_shift['date_block_num'] = train_shift['date_block_num'] + month_shift
foo = lambda x: '{}_lag_{}'.format(x, month_shift)if x in cols_to_rename else x
train_shift = train_shift.rename(columns=foo)
 all_data = pd.merge(all_data, train_shift, on=index_cols, how='left' ).fillna(0)
 del train_shift
all_data = all_data[all_data['date_block_num'] >= 12]
fit_cols = [col for col in all_data.columns if col[-1] in [str(item)for item in shift_range]]
to_drop_cols = list(set(list(all_data.columns)) -(set(fit_cols)|set(index_cols)))+ ['date_block_num']
item_category_mapping = items[['item_id','item_category_id']].drop_duplicates()
all_data = pd.merge(all_data, item_category_mapping, how='left', on='item_id')
 all_data = downcast_dtypes(all_data)
 gc.collect() ;<groupby>
df_train = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv")
 print("train data shape --",df_train.shape)
 print("test data shape --",df_test.shape )
item_target_enc_na =.3343
cumsum = all_data.groupby('item_id')['target'].cumsum() - all_data['target']
cumcnt = all_data.groupby('item_id')['target'].cumcount()
all_data['item_target_enc'] = cumsum/cumcnt
all_data['item_target_enc'].fillna(item_target_enc_na,inplace=True)
 corr = np.corrcoef(all_data['target'].values, all_data['item_target_enc'])[0][1]
print(corr )<groupby>
df_train["question_text"] = df_train["question_text"].apply(lambda x: x.replace('.',' fullstop '))
df_train["question_text"] = df_train["question_text"].apply(lambda x: x.replace('?',' endofquestion '))
df_train["question_text"] = df_train["question_text"].apply(lambda x: x.replace(',',' comma '))
df_train["question_text"] = df_train["question_text"].apply(lambda x: x.replace('!',' exclamationmark '))
df_train["question_text"] = df_train["question_text"].apply(lambda x: x.replace('-',' hyphen '))
df_train["question_text"] = df_train["question_text"].apply(lambda x: x.replace('/',' backslash '))
item_target_enc_na =.3343
cumsum = all_data.groupby('shop_id')['target'].cumsum() - all_data['target']
cumcnt = all_data.groupby('shop_id')['target'].cumcount()
all_data['shop_id_enc'] = cumsum/cumcnt
all_data['shop_id_enc'].fillna(item_target_enc_na,inplace=True)
 corr = np.corrcoef(all_data['target'].values, all_data['shop_id_enc'])[0][1]
print(corr )<feature_engineering>
df_train["question_text"] = df_train["question_text"].apply(lambda x: x.replace('fullstop','.'))
df_train["question_text"] = df_train["question_text"].apply(lambda x: x.replace('endofquestion','?'))
df_train["question_text"] = df_train["question_text"].apply(lambda x: x.replace('comma',','))
df_train["question_text"] = df_train["question_text"].apply(lambda x: x.replace('exclamationmark','!'))
df_train["question_text"] = df_train["question_text"].apply(lambda x: x.replace('hyphen','-'))
df_train["question_text"] = df_train["question_text"].apply(lambda x: x.replace('backslash','/'))
sales_train['month'] = sales_train['date_block_num']%12+1
sales_train[['date','month','date_block_num']].sample(5)
<feature_engineering>
df_test["question_text"] = df_test["question_text"].apply(lambda x: x.replace('.',' fullstop '))
df_test["question_text"] = df_test["question_text"].apply(lambda x: x.replace('?',' endofquestion '))
df_test["question_text"] = df_test["question_text"].apply(lambda x: x.replace(',',' comma '))
df_test["question_text"] = df_test["question_text"].apply(lambda x: x.replace('!',' exclamationmark '))
df_test["question_text"] = df_test["question_text"].apply(lambda x: x.replace('-',' hyphen '))
df_test["question_text"] = df_test["question_text"].apply(lambda x: x.replace('/',' backslash '))
all_data['month'] = all_data['date_block_num']%12+1<categorify>
df_test["question_text"] = df_test["question_text"].apply(lambda x: x.replace('fullstop','.'))
df_test["question_text"] = df_test["question_text"].apply(lambda x: x.replace('endofquestion','?'))
df_test["question_text"] = df_test["question_text"].apply(lambda x: x.replace('comma',','))
df_test["question_text"] = df_test["question_text"].apply(lambda x: x.replace('exclamationmark','!'))
df_test["question_text"] = df_test["question_text"].apply(lambda x: x.replace('hyphen','-'))
df_test["question_text"] = df_test["question_text"].apply(lambda x: x.replace('backslash','/'))
all_data['item_category_id'] = all_data['item_id'].map(items.set_index('item_id')['item_category_id'] )<count_missing_values>
df_combined = pd.concat([df_train,df_test],axis=0)
 print("combined shape ",df_combined.shape )
all_data['item_category_id'].isna().sum()<feature_engineering>
embed_size = 300
max_features = 60000
maxlen = 60
total_X = df_combined["question_text"].values
item_target_enc_na = 0
cumsum = all_data.groupby('item_category_id')['target'].cumsum() - all_data['target']
cumcnt = all_data.groupby('item_category_id')['target'].cumcount()
item_category_id_enc = cumsum/cumcnt
item_category_id_enc.fillna(item_target_enc_na,inplace=True)
 all_data['item_category_id_enc'] = item_category_id_enc
corr = np.corrcoef(all_data['target'].values, item_category_id_enc)[0][1]
print(corr )<count_unique_values>
tokenizer = Tokenizer(num_words=max_features,filters='"
',)
 tokenizer.fit_on_texts(list(total_X))
words = ' '.join(shops['shop_name'] ).split(' ')
 c = Counter(words)
 c.most_common(10 )<merge>
WORDS = tokenizer.word_counts
print(len(WORDS))
shop_by_store = sales_train.groupby('shop_id',as_index=False)['item_cnt_day'].sum()
shop_by_store = shop_by_store.merge(shops, on='shop_id')
 print(shop_by_store['shop_name'].isna().sum())
 shop_by_store.head()<feature_engineering>
train_X = df_train["question_text"].values
test_X = df_test["question_text"].values
shop_by_store['name_array'] = shop_by_store['shop_name'].str.split(' ')
 top_words = [x for x,y in c.most_common(6)]
for w in top_words:
shop_by_store[w] = shop_by_store['shop_name'].map(lambda x: 1 if w in x else 0)
<groupby>
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X )
for w in top_words:
print(shop_by_store.groupby(by=w)['item_cnt_day'].mean() )<merge>
train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = df_train['target'].values
top_words = top_words[0:4]
shops['name_array'] = shops['shop_name'].str.split(' ')
 for w in top_words:
shops[w] = shops['shop_name'].map(lambda x: 1 if w in x else 0)
 all_data = pd.merge(all_data,shops[['shop_id']+top_words],on='shop_id',how='left')
 print(all_data[top_words].isna().sum())
 all_data.head()<prepare_x_and_y>
def get_embeddings(embedtype):
if embedtype is "glove":
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
elif embedtype is "fastext":
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
elif embedtype is "paragram":
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE,encoding="latin")if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
vocab_size = all_embs.shape[0]
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
print("embed size-->",embed_size)
 print("total words in embeddings-->",vocab_size)
 print("total words in data-->",len(word_index))
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
count_common_words =0
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
count_common_words = count_common_words+1
print(count_common_words," common words fount in ",embedtype)
 print("{0}% common words found in {1}".format(( count_common_words*100/len(word_index)) ,embedtype))
return embedding_matrix
leaking_columns = ['median_item_price','date_block_num','target','target_shop','target_item','trips','trips_shop','trips_item']
X_train = all_data.loc[all_data['date_block_num'] < 33].drop(leaking_columns, axis=1)
 X_val = all_data.loc[all_data['date_block_num'] == 33].drop(leaking_columns, axis=1)
 X_test = all_data.loc[all_data['date_block_num'] == 34].drop(leaking_columns, axis=1)
 y_train = all_data.loc[all_data['date_block_num'] < 33,'target'].values
y_val = all_data.loc[all_data['date_block_num'] == 33,'target'].values
del all_data
gc.collect() ; 
<compute_test_metric>
embedding_glove = get_embeddings(embedtype="glove" )
def validation_function(y_pred,y_true):
print(f'rmse before [0,20] clipping: {mean_squared_error(y_true, y_pred)}')
 y_pred = y_pred.clip(0,20)
 y_true = y_true.clip(0,20)
 print(f'rmse after [0,20] clipping: {mean_squared_error(y_true, y_pred)}')
 return 
<train_model>
embedding_paragram = get_embeddings(embedtype="paragram" )
lr = LinearRegression()
lr.fit(X_train.values, y_train)
 pred_lr = lr.predict(X_val.values)
 print('Test R-squared for linreg is %f' % r2_score(y_val, pred_lr))
validation_function(y_val,pred_lr )<train_model>
mean_gl_par_embedding = np.mean([embedding_glove,embedding_paragram],axis=0)
 print("mean glove paragram embedding shape--> ",mean_gl_par_embedding.shape )
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_eval = lgb.Dataset(X_val, y_val, reference=lgb_train)
 params = {
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': {'root_mean_squared_error'},
'num_leaves': 31,
'learning_rate': 0.05,
'feature_fraction': 0.9,
'bagging_fraction': 0.8,
'bagging_freq': 5,
'verbose': 10
} 
<train_model>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
gbm = lgb.train(params,
lgb_train,
num_boost_round=15,
valid_sets=[lgb_train,lgb_eval],
valid_names=['train','val'],
early_stopping_rounds=5)
 print(gbm.pandas_categorical)
 lgb.plot_importance(gbm,figsize=(10,10)) ;<train_model>
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr() )
categorical_features = ['shop_id','item_id','item_category_id','new_item']+top_words
lgb_train = lgb.Dataset(X_train, y_train,categorical_feature=categorical_features)
 lgb_eval = lgb.Dataset(X_val, y_val, reference=lgb_train,categorical_feature=categorical_features)
 gbm2 = lgb.train(params,
lgb_train,
num_boost_round=15,
valid_sets=[lgb_train,lgb_eval],
valid_names=['train','val'],
categorical_feature = categorical_features,
early_stopping_rounds=5)
 print(gbm2.pandas_categorical)
 lgb.plot_importance(gbm2,figsize=(10,10)) ; 
<create_dataframe>
def build_model() :
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[mean_gl_par_embedding],trainable=False )(inp)
 x = SpatialDropout1D(rate=0.1 )(x)
 x1 = Bidirectional(CuDNNGRU(200, return_sequences=True))(x)
 x2 = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 atten_1 = Attention(maxlen )(x1)
 atten_2 = Attention(maxlen )(x2)
 x = concatenate([atten_1,atten_2])
 x = Dense(128, activation="relu" )(x)
 x = Dropout(0.2 )(x)
 x = BatchNormalization()(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam() ,)
 return model
lgb_train = lgb.Dataset(X_train, y_train,categorical_feature=categorical_features)
 lgb_eval = lgb.Dataset(X_val, y_val, reference=lgb_train,categorical_feature=categorical_features)
 lgb_params = {
'feature_fraction': 0.75,
'metric': 'rmse',
'nthread':1,
'min_data_in_leaf': 2**7,
'bagging_fraction': 0.75,
'learning_rate': 0.03,
'objective': 'mse',
'bagging_seed': 2**7,
'num_leaves': 2**7,
'bagging_freq':1,
'verbose':10
}
model = lgb.train(lgb_params,
lgb_train,
num_boost_round=1000,
valid_sets=[lgb_train,lgb_eval],
valid_names=['train','val'],
categorical_feature = categorical_features,
early_stopping_rounds=5)
 print(model.pandas_categorical)
 lgb.plot_importance(model,figsize=(10,10)) ; 
<predict_on_test>
def f1_smart(y_true, y_pred):
args = np.argsort(y_pred)
 tp = y_true.sum()
fs =(tp - np.cumsum(y_true[args[:-1]])) / np.arange(y_true.shape[0] + tp - 1, tp, -1)
 res_idx = np.argmax(fs)
 return 2 * fs[res_idx],(y_pred[args[res_idx]] + y_pred[args[res_idx + 1]])/ 2
pred_lgb = model.predict(X_val)
 print('Test R-squared for LightGBM is %f' % r2_score(y_val, pred_lgb))
validation_function(y_val,pred_lgb )<choose_model_class>
kfold = StratifiedKFold(n_splits=5, random_state=1990, shuffle=True)
 bestscore = []
y_test = np.zeros(( test_X.shape[0],))
filepath="weights_best_mean.h5"
for i,(train_index, valid_index)in enumerate(kfold.split(train_X, train_y)) :
X_train, X_val, Y_train, Y_val = train_X[train_index], train_X[valid_index], train_y[train_index], train_y[valid_index]
checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='min')
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.6, patience=1, min_lr=0.0001, verbose=2)
 earlystopping = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=2, verbose=2, mode='auto')
 callbacks = [clr,]
model = build_model()
if i == 0:print(model.summary())
 model.fit(X_train, Y_train, batch_size=512, epochs=2, validation_data=(X_val, Y_val), verbose=1, callbacks=[checkpoint,clr,])
 model.load_weights(filepath)
 y_pred = model.predict([X_val], batch_size=1024, verbose=1)
 y_test += np.squeeze(model.predict([test_X], batch_size=1024, verbose=2)) /5
f1, threshold = f1_smart(np.squeeze(Y_val), np.squeeze(y_pred))
print('Optimal F1: {:.4f} at threshold: {:.4f}'.format(f1, threshold))
bestscore.append(threshold )
X_train_level2 = np.c_[model.predict(X_train), lr.predict(X_train.values)]
lr2 = LinearRegression()
lr2.fit(X_train_level2, y_train)
 X_val_level2 = np.c_[model.predict(X_val), lr.predict(X_val.values)]
pred_lr2 = lr2.predict(X_val_level2)
 validation_function(y_val,pred_lr2)
<predict_on_test>
print("mean threshold--> ",np.mean(bestscore))
<sort_values><EOS> 
print(y_test.shape)
 pred_test_y =(y_test>np.mean(bestscore)).astype(int)
 out_df = pd.DataFrame({"qid":df_test["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )