Kaggle competition: ML Challenge
%%time
historical_transactions = pd.get_dummies(historical_transactions, columns=['category_2', 'category_3'])
 new_transactions = pd.get_dummies(new_transactions, columns=['category_2', 'category_3'])
 historical_transactions = reduce_mem_usage(historical_transactions)
 new_transactions = reduce_mem_usage(new_transactions)
 agg_fun = {'authorized_flag': ['sum', 'mean']}
auth_mean = historical_transactions.groupby(['card_id'] ).agg(agg_fun)
 auth_mean.columns = ['_'.join(col ).strip() for col in auth_mean.columns.values]
auth_mean.reset_index(inplace=True)
 authorized_transactions = historical_transactions[historical_transactions['authorized_flag'] == 1]
historical_transactions = historical_transactions[historical_transactions['authorized_flag'] == 0]
gc.collect()<feature_engineering>
np.random.seed(10)
%%time
historical_transactions['purchase_month'] = historical_transactions['purchase_date'].dt.month
authorized_transactions['purchase_month'] = authorized_transactions['purchase_date'].dt.month
new_transactions['purchase_month'] = new_transactions['purchase_date'].dt.month
gc.collect()<data_type_conversions>
def trim_all_columns(df):
trim_strings = lambda x: x.strip() if isinstance(x, str)else x
return df.applymap(trim_strings )
%%time
def aggregate_transactions(history):
history.loc[:, 'purchase_date'] = pd.DatetimeIndex(history['purchase_date'] ).\
astype(np.int64)* 1e-9
agg_func = {
'category_1': ['sum', 'mean'],
'category_2_1.0': ['mean'],
'category_2_2.0': ['mean'],
'category_2_3.0': ['mean'],
'category_2_4.0': ['mean'],
'category_2_5.0': ['mean'],
'category_3_A': ['mean'],
'category_3_B': ['mean'],
'category_3_C': ['mean'],
'merchant_id': ['nunique'],
'merchant_category_id': ['nunique'],
'state_id': ['nunique'],
'city_id': ['nunique'],
'subsector_id': ['nunique'],
'purchase_amount': ['sum', 'mean', 'max', 'min', 'std'],
'installments': ['sum', 'mean', 'max', 'min', 'std'],
'purchase_month': ['mean', 'max', 'min', 'std'],
'purchase_date': [np.ptp, 'min', 'max'],
'month_lag': ['min', 'max']
}
agg_history = history.groupby(['card_id'] ).agg(agg_func)
 agg_history.columns = ['_'.join(col ).strip() for col in agg_history.columns.values]
agg_history.reset_index(inplace=True)
 df =(history.groupby('card_id')
.size() 
.reset_index(name='transactions_count'))
agg_history = pd.merge(df, agg_history, on='card_id', how='left')
 return agg_history
gc.collect()<groupby>
header_list = ["AAGE", "ACLSWKR", "ADTIND", "ADTOCC", "AHGA","AHRSPAY","AHRSCOL",
"AMARITL","AMJIND","AMJOCC","ARACE","AREORGN","ASEX","AUNMEM",
"AUNTYPE","AWKSTAT","CAPGAIN","CAPLOSS","DIVVAL", "FILESTAT",
"GRINREG","GRINST","HHDFMX","HHDREL", "MARSUPWT","MIGMTR1",
"MIGMTR3","MIGMTR4","MIGSAME", "MIGSUN","NOEMP","PARENT",
"PEFNTVTY","PEMNTVTY","PENATVTY","PRCITSHP","SEOTR",
"VETQVA","VETYN","WKSWORK","YEAR","INCCAT"]
%%time
def aggregate_per_month(history):
grouped = history.groupby(['card_id', 'month_lag'])
 agg_func = {
'purchase_amount': ['count', 'sum', 'mean', 'min', 'max', 'std'],
'installments': ['count', 'sum', 'mean', 'min', 'max', 'std'],
}
intermediate_group = grouped.agg(agg_func)
 intermediate_group.columns = ['_'.join(col ).strip() for col in intermediate_group.columns.values]
intermediate_group.reset_index(inplace=True)
 final_group = intermediate_group.groupby('card_id' ).agg(['mean', 'std'])
 final_group.columns = ['_'.join(col ).strip() for col in final_group.columns.values]
final_group.reset_index(inplace=True)
 return final_group
final_group = aggregate_per_month(historical_transactions)
 final_group[:10]
gc.collect()<merge>
d_train=pd.read_csv('/kaggle/input/ml-challenge-week6/census-income.data',index_col=False,names=header_list)
 d_test=pd.read_csv('/kaggle/input/ml-challenge-week6/census-income.test',index_col=False,names=header_list )
%%time
train = pd.merge(train, history, on='card_id', how='left')
 test = pd.merge(test, history, on='card_id', how='left')
 train = pd.merge(train, authorized, on='card_id', how='left')
 test = pd.merge(test, authorized, on='card_id', how='left')
 train = pd.merge(train, new, on='card_id', how='left')
 test = pd.merge(test, new, on='card_id', how='left')
 train = pd.merge(train, final_group, on='card_id', how='left')
 test = pd.merge(test, final_group, on='card_id', how='left')
 train = pd.merge(train, auth_mean, on='card_id', how='left')
 test = pd.merge(test, auth_mean, on='card_id', how='left')
 print("Train Shape:", train.shape)
 print("Test Shape:", test.shape)
 gc.collect()<define_variables>
d_test=trim_all_columns(d_test)
 d_train=trim_all_columns(d_train )
features = [c for c in train.columns if c not in ['card_id', 'first_active_month']]
categorical_feats = [c for c in features if 'feature_' in c]<init_hyperparams>
label = {'50000+.':1, '- 50000.':0}
d_train['INCCAT']=d_train['INCCAT'].map(label)
 d_test['INCCAT']=d_test['INCCAT'].map(label)
 d_test['AREORGN']=d_test['AREORGN'].fillna('NA')
 d_test=d_test.fillna('?')
param = {'num_leaves': 31,
'min_data_in_leaf': 32,
'objective':'regression',
'max_depth': -1,
'learning_rate': 0.005,
"min_child_samples": 20,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9 ,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.1,
"nthread": 4,
"verbosity": -1}<split>
folds = KFold(n_splits=5, shuffle=True, random_state=15)
 oof = np.zeros(len(train))
predictions = np.zeros(len(test))
start = time.time()
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train.values, target.values)) :
print("fold n°{}".format(fold_))
trn_data = lgb.Dataset(train.iloc[trn_idx][features], label=target.iloc[trn_idx], categorical_feature=categorical_feats)
 val_data = lgb.Dataset(train.iloc[val_idx][features], label=target.iloc[val_idx], categorical_feature=categorical_feats)
 num_round = 10000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=100, early_stopping_rounds = 200)
 oof[val_idx] = clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(mean_squared_error(oof, target)**0.5))<init_hyperparams>
d_train.loc[(d_train['AWKSTAT']=='Children or Armed Forces')&(d_train['AAGE']<=18),'AWKSTAT']='Children'
d_test.loc[(d_test['AWKSTAT']=='Children or Armed Forces')&(d_test['AAGE']<=18),'AWKSTAT']='Children'
numb=d_train.groupby(['AWKSTAT','AMJIND'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb=numb.reset_index()
print("train=", d_train.shape," test=", d_test.shape)
lgbparam = {'num_leaves': 31,
'boosting_type': 'rf',
'min_data_in_leaf': 30,
'objective':'regression',
'max_depth': -1,
'learning_rate': 0.01,
"min_child_samples": 20,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9 ,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.1,
"verbosity": -1,
"nthread": 4,
"random_state": 4590}<create_dataframe>
pd.set_option('display.max_rows', None)
 numb=d_train.groupby(['AAGE'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb=numb.reset_index()
folds = RepeatedKFold(n_splits=5, n_repeats=2, random_state=4520)
 oof_lgb = np.zeros(len(train))
predictions_lgb = np.zeros(len(test))
start = time.time()
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train.values, target.values)) :
print("fold n°{}".format(fold_))
trn_data = lgb.Dataset(train.iloc[trn_idx][features], label=target.iloc[trn_idx], categorical_feature=categorical_feats)
 val_data = lgb.Dataset(train.iloc[val_idx][features], label=target.iloc[val_idx], categorical_feature=categorical_feats)
 num_round = 11000
clf = lgb.train(lgbparam, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=100, early_stopping_rounds = 100)
 oof_lgb[val_idx] = clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions_lgb += clf.predict(test[features], num_iteration=clf.best_iteration)/(5 * 2)
 print("CV score: {:<8.5f}".format(mean_squared_error(oof_lgb, target)**0.5))<save_to_csv>
d_train.loc[d_train['ACLSWKR']=="Never worked",['INCCAT']]=0
d_train.loc[d_train['ACLSWKR']=="Without pay",['INCCAT']]=0
sub_df = pd.DataFrame({"card_id":test["card_id"].values})
 sub_df["target"] = predictions
sub_df.to_csv("submit_lgb.csv", index=False)
 sub_df1 = pd.DataFrame({"card_id":test["card_id"].values})
 sub_df1["target"] = predictions_lgb
sub_df1.to_csv("submit_lgb1.csv", index=False )<split>
numb=d_train.groupby(['ACLSWKR'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
a=numb.copy()
dummy_le=LabelEncoder()
a=a.reset_index()
a.iloc[:, 0]=dummy_le.fit_transform(numb.iloc[:, 0])
 numb=numb.sort_values(numb.columns[2] ).reset_index()
le_ACLSWKR = LabelEncoder()
le_ACLSWKR.classes_=numb.iloc[:, 0]
numb.iloc[:, 0] = le_ACLSWKR.transform(numb.iloc[:, 0])
 plt.figure(1)
 plt.subplot(211)
 plt.plot(a['ACLSWKR'],a['Ratio'])
 plt.subplot(212)
 plt.plot(numb['ACLSWKR'],numb['Ratio'])
train_stack = np.vstack([oof,oof_lgb] ).transpose()
test_stack = np.vstack([predictions,predictions_lgb] ).transpose()
folds = RepeatedKFold(n_splits=5,n_repeats=1,random_state=4520)
 oof_stack = np.zeros(train_stack.shape[0])
 predictions_stack = np.zeros(test_stack.shape[0])
 for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_stack, target)) :
print("fold n°{}".format(fold_))
trn_data, trn_y = train_stack[trn_idx], target.iloc[trn_idx].values
val_data, val_y = train_stack[val_idx], target.iloc[val_idx].values
print("-" * 10 + "Stacking " + str(fold_)+ "-" * 10)
 clf = BayesianRidge()
clf.fit(trn_data, trn_y)
 oof_stack[val_idx] = clf.predict(val_data)
 predictions_stack += clf.predict(test_stack)/ 5
np.sqrt(mean_squared_error(target.values, oof_stack))<save_to_csv>
d_train.loc[:, 'ACLSWKR'] = le_ACLSWKR.transform(d_train.loc[:, 'ACLSWKR'])
 d_test.loc[:, 'ACLSWKR'] = le_ACLSWKR.transform(d_test.loc[:, 'ACLSWKR'] )
sample_submission = pd.read_csv('.. /input/elo-merchant-category-recommendation/sample_submission.csv')
 sample_submission['target'] = predictions_stack
sample_submission.to_csv('Bayesian_Ridge_Stacking.csv', index=False )<save_to_csv>
numb=d_train.groupby(['AHGA'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb.sort_values(numb.columns[2] ).reset_index()
sample_submission = pd.read_csv('.. /input/elo-merchant-category-recommendation/sample_submission.csv')
 sample1 = pd.read_csv(".. /input/elo-blending/3.695.csv")
 sample2 = pd.read_csv(".. /input/elo-blending/combining_submission(1 ).csv")
 sample_submission['target'] = predictions * 0.5 + predictions_lgb * 0.5
sample_submission.to_csv("Blend1.csv", index = False)
 sample_submission['target'] = sample_submission['target'] * 0.2 + sample1['target'] * 0.2 + sample2['target'] * 0.6
sample_submission.to_csv('Blend2.csv', index=False )<categorify>
numb=d_train.groupby(['AHGA'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb=numb.sort_values(numb.columns[2])
 numb=numb.reset_index()
le_AHGA = LabelEncoder()
le_AHGA.classes_=numb.iloc[:, 0]
dim = 300
num_words = 50000
max_len = 100
print('Fiting tokenizer')
 tokenizer = Tokenizer(num_words=num_words)
 tokenizer.fit_on_texts(df['question_text'])
 print('text to sequence')
 x_train = tokenizer.texts_to_sequences(df['question_text'])
 print('pad sequence')
 x_train = pad_sequences(x_train,maxlen=max_len)
 y_train = df['target'].values
print(x_train.shape)
 print(y_train.shape )<compute_train_metric>
d_train.loc[:, 'AHGA'] = le_AHGA.transform(d_train.loc[:, 'AHGA'])
 d_test.loc[:, 'AHGA'] = le_AHGA.transform(d_test.loc[:, 'AHGA'] )
print('Glove...')
 def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_glov = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_glov[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()<statistical_test>
numb=d_train.groupby(['AHRSCOL'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb=numb.sort_values(numb.columns[2])
 numb=numb.reset_index()
le_AHRSCOL = LabelEncoder()
le_AHRSCOL.classes_=numb.iloc[:, 0]
print('Para...')
 EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_para = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_para[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()<statistical_test>
d_train.loc[:, 'AHRSCOL'] = le_AHRSCOL.transform(d_train.loc[:, 'AHRSCOL'])
 d_test.loc[:, 'AHRSCOL'] = le_AHRSCOL.transform(d_test.loc[:, 'AHRSCOL'] )
print('Wiki...')
 EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_wiki = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_wiki[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_wiki.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()<feature_engineering>
numb=d_train.groupby(['AMARITL'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb=numb.sort_values(numb.columns[2])
 numb=numb.reset_index()
le_AMARITL = LabelEncoder()
le_AMARITL.classes_=numb.iloc[:, 0]
d_train.loc[:, 'AMARITL'] = le_AMARITL.transform(d_train.loc[:, 'AMARITL'])
 d_test.loc[:, 'AMARITL'] = le_AMARITL.transform(d_test.loc[:, 'AMARITL'] )
matrixes = [embedding_matrix_glov,embedding_matrix_wiki,embedding_matrix_para]
matrix = np.mean(matrixes,axis=0)
 del embedding_matrix_glov,embedding_matrix_wiki,embedding_matrix_para
gc.collect()<load_from_csv>
numb=d_train.groupby(['AMJIND','ADTIND'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb=numb.reset_index()
numb.sort_values(numb.columns[0])
 nu=d_train.groupby(['AMJIND'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 nu['Ratio']=nu['Nr']/nu['Tot']
nu=nu.reset_index()
for index, row in numb.iterrows() :
if numb.loc[numb['ADTIND']==row['ADTIND'],'Tot'].values[0]<300:
numb.loc[numb['ADTIND']==row['ADTIND'],'Ratio']=nu.loc[nu['AMJIND']==row['AMJIND']]['Ratio'].values[0]
numb.loc[numb['AMJIND']=='Armed Forces','Ratio']=nu.loc[nu['AMJIND']=='Manufacturing-durable goods']['Ratio'].values[0]
for index, row in numb.iterrows() :
d_train.loc[d_train['ADTIND']==row['ADTIND'],'AMJIND']=row['Ratio']
d_test.loc[d_test['ADTIND']==row['ADTIND'],'AMJIND']=row['Ratio']
print('Loading test data...')
 df_final = pd.read_csv('.. /input/test.csv')
 df_final["question_text"].fillna("_
x_test=tokenizer.texts_to_sequences(df_final['question_text'])
 x_test = pad_sequences(x_test,maxlen=max_len)
 print('Test data loaded:',x_test.shape )<import_modules>
pd.set_option('display.max_rows', 50)
 numb=d_train.groupby(['AMJOCC','ADTOCC'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb=numb.reset_index()
numb.sort_values(numb.columns[0])
 nu=d_train.groupby(['AMJOCC'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 nu['Ratio']=nu['Nr']/nu['Tot']
nu=nu.reset_index()
for index, row in numb.iterrows() :
if numb.loc[numb['ADTOCC']==row['ADTOCC'],'Tot'].values[0]<300:
numb.loc[numb['ADTOCC']==row['ADTOCC'],'Ratio']=nu.loc[nu['AMJOCC']==row['AMJOCC']]['Ratio'].values[0]
numb.loc[numb['AMJOCC']=='Armed Forces','Ratio']=nu.loc[nu['AMJOCC']=='Technicians and related support']['Ratio'].values[0]
numb
for index, row in numb.iterrows() :
d_train.loc[d_train['ADTOCC']==row['ADTOCC'],'AMJOCC']=row['Ratio']
d_test.loc[d_test['ADTOCC']==row['ADTOCC'],'AMJOCC']=row['Ratio']
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
 class Metrics(Callback):
def on_train_begin(self, logs={}):
self.val_f1s = []
def _threshold_search(self,y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
def on_epoch_end(self, epoch, logs={}):
y_val = self.validation_data[1]
y_pred = self.model.predict(self.validation_data[0])
 with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 search_result = self._threshold_search(y_val, y_pred)
 _val_f1 = search_result['f1']
self.val_f1s.append(_val_f1)
 print('— val_f1_on_epoch_end: %f '%(_val_f1))
return
def get_model(adam, trainable=False):
inp1 = Input(shape=(max_len,))
emb = Embedding(num_words, dim, weights=[matrix],trainable = trainable, )(inp1)
 filters = []
for f in [3,4,5]:
conv = Conv1D(64,f, activation='elu' )(emb)
 pool = GlobalMaxPooling1D()(conv)
 filters.append(pool)
 x = concatenate(filters)
 x = Dense(max_len, activation="relu" )(x)
 x = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=inp1, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy'])
 print(model.summary())
 return model
def get_opt() :
adam = optimizers.Adam()
print('LR:',K.eval(adam.lr))
return adam
def get_batch_size(opt):
batch_size = int(x_train.shape[0]*K.eval(opt.lr))
print('Batch size = ',batch_size)
 return batch_size
patience = 2
best_model=None
all_results = {}
train_meta = np.zeros(y_train.shape)
 test_meta = np.zeros(x_test.shape[0])
 splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=14 ).split(x_train, y_train))
for idx,(train_idx, valid_idx)in enumerate(splits):
print('----'+str(idx)+'-----')
 X_train1 = x_train[train_idx]
y_train1 = y_train[train_idx]
X_val = x_train[valid_idx]
y_val = y_train[valid_idx]
model_file = 'model_'+str(idx)+'.h5'
modelcheck = ModelCheckpoint(model_file,save_best_only=True)
 stop = EarlyStopping(patience=patience)
 reduce = CyclicLR(base_lr=0.001,
max_lr=0.004,
step_size=300.,
mode='exp_range',
gamma=0.99994)
 metrics = Metrics()
adam = get_opt()
batch_size = get_batch_size(adam)
 model = get_model(adam)
 history = model.fit(X_train1,y_train1,
batch_size=batch_size,
validation_data=(X_val,y_val),
epochs=100,
callbacks=[modelcheck,stop,reduce,metrics],
verbose=2)
 print('Pretraining finished, unfreezing embeddings layer...')
 model = get_model(adam,True)
 model.load_weights(model_file)
 modelcheck = ModelCheckpoint(model_file,save_best_only=True)
 stop = EarlyStopping(patience=patience)
 reduce = CyclicLR(base_lr=0.001,
max_lr=0.004,
step_size=300.,
mode='exp_range',
gamma=0.99994)
 metrics = Metrics()
history = model.fit(X_train1,y_train1,
batch_size=batch_size,
validation_data=(X_val,y_val),
epochs=100,
callbacks=[modelcheck,stop,reduce,metrics],
verbose=2)
 print('training finished...')
 model.load_weights(model_file)
 y_pred = model.predict(X_val,batch_size=batch_size, verbose=1)
 train_meta[valid_idx] = y_pred.reshape(-1)
 search_result = threshold_search(y_val, y_pred)
 print(search_result)
 y_pred = y_pred>search_result['threshold']
y_pred = y_pred.astype(int)
 print('RESULTS ON VALIDATION SET:
',classification_report(y_val,y_pred))
all_results[model_file] = search_result['f1']
y_pred = model.predict(x_test,batch_size=batch_size, verbose=1)
 test_meta += y_pred.reshape(-1)/ len(splits)
 if best_model is None or best_model['f1'] < search_result['f1']:
best_model={'model':model_file,'f1':search_result['f1']}
print('-'*80)
 print(all_results)
 print('-'*80)
 print(best_model)
 print('-'*80)
 train_meta = train_meta[train_meta > -1]
y_meta = train_meta[train_meta > -1] 
<save_to_csv>
numb=d_train.groupby(['ARACE'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb=numb.sort_values(numb.columns[2])
 numb=numb.reset_index()
le_ARACE = LabelEncoder()
le_ARACE.classes_=numb.iloc[:, 0]
d_train.loc[:, 'ARACE'] = le_ARACE.transform(d_train.loc[:, 'ARACE'])
 d_test.loc[:, 'ARACE'] = le_ARACE.transform(d_test.loc[:, 'ARACE'] )
search_result = threshold_search(y_train, train_meta)
 print(search_result)
 df_subm = pd.DataFrame()
df_subm['qid'] = df_final.qid
df_subm['prediction'] = test_meta > 0.25
print(df_subm.head())
 df_subm.to_csv('submission.csv', index=False )<set_options>
numb=d_train.groupby(['AREORGN'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb=numb.sort_values(numb.columns[2])
 numb=numb.reset_index()
le_AREORGN = LabelEncoder()
le_AREORGN.classes_=numb.iloc[:, 0]
d_train.loc[:, 'AREORGN'] = le_AREORGN.transform(d_train.loc[:, 'AREORGN'])
 d_test.loc[:, 'AREORGN'] = le_AREORGN.transform(d_test.loc[:, 'AREORGN'] )
warnings.simplefilter(action='ignore', category=FutureWarning )<load_from_csv>
numb=d_train.groupby(['ASEX'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb=numb.sort_values(numb.columns[2])
 numb=numb.reset_index()
le_ASEX = LabelEncoder()
le_ASEX.classes_=numb.iloc[:, 0]
d_train.loc[:, 'ASEX'] = le_ASEX.transform(d_train.loc[:, 'ASEX'])
 d_test.loc[:, 'ASEX'] = le_ASEX.transform(d_test.loc[:, 'ASEX'] )
def load_data() :
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 list_sentences_train = list(train_df["question_text"].fillna("NAN_WORD" ).values)
 list_sentences_test = list(test_df["question_text"].fillna("NAN_WORD" ).values)
 return list_sentences_train, list_sentences_test<split>
numb=d_train.groupby(['AUNMEM','AUNTYPE'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb=numb.sort_values(numb.columns[2])
 numb=numb.reset_index()
le_AUN = LabelEncoder()
le_AUN.classes_=numb.iloc[:, 0] +" "+ numb.iloc[:, 1]
d_train.loc[:,'AUNMEM'] = le_AUN.transform(( d_train.loc[:, 'AUNMEM'] + " " + d_train.loc[:,'AUNTYPE']))
d_test.loc[:,'AUNMEM'] = le_AUN.transform(( d_test.loc[:, 'AUNMEM'] + " " + d_test.loc[:,'AUNTYPE']))
d_train=d_train.drop(['AUNTYPE'],axis=1)
 d_test=d_test.drop(['AUNTYPE'],axis=1)
list_sentences_train, list_sentences_test = load_data()<load_from_csv>
numb=d_train.groupby(['AWKSTAT'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb=numb.sort_values(numb.columns[2])
 numb=numb.reset_index()
le_AWKSTAT = LabelEncoder()
le_AWKSTAT.classes_=numb.iloc[:, 0]
d_train.loc[:, 'AWKSTAT'] = le_AWKSTAT.transform(d_train.loc[:, 'AWKSTAT'])
 d_test.loc[:, 'AWKSTAT'] = le_AWKSTAT.transform(d_test.loc[:, 'AWKSTAT'] )
tmp = Counter([i for i in pd.read_csv(".. /input/train.csv")['target'].tolist() ])
 print("The percentage of insincere questions in train set is {:.2%}.".format(tmp[1]/(tmp[1]+tmp[0])))
 train_1_prc = tmp[1]/(tmp[1]+tmp[0])
 del(tmp )<load_from_csv>
numb=d_train.groupby(['CAPGAIN'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb=numb.reset_index()
numb=numb.sort_values(numb.columns[3])
 d_train.loc[d_train['CAPGAIN']<=3000,'CAPGAIN']=0
d_test.loc[d_test['CAPGAIN']<=3000,'CAPGAIN']=0
d_train.loc[(d_train['CAPGAIN']>3000)&(d_train['CAPGAIN']<10000),'CAPGAIN']=1
d_test.loc[(d_test['CAPGAIN']>3000)&(d_test['CAPGAIN']<10000),'CAPGAIN']=1
d_train.loc[d_train['CAPGAIN']>=10000,'CAPGAIN']=2
d_test.loc[d_test['CAPGAIN']>=10000,'CAPGAIN']=2
%matplotlib inline
sns.set(rc={'figure.figsize':(12,6)})
 sns.distplot(np.array([len(i.split())for i in list_sentences_train])[np.array([i==1 for i in pd.read_csv(".. /input/train.csv")['target'].tolist() ])])
 plt.title('Distribution of the length of the question_text - only insincere questions' )<define_variables>
numb=d_train.groupby(['DIVVAL'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb=numb.reset_index()
numb=numb.sort_values(numb.columns[0])
 d_train['DIVCAT']=0
d_test['DIVCAT']=0
d_train.loc[d_train['DIVVAL']<700,'DIVCAT']=0
d_test.loc[d_test['DIVVAL']<700,'DIVCAT']=0
d_train.loc[(d_train['DIVVAL']>=700)&(d_train['DIVVAL']<=11000),'DIVCAT']=1
d_test.loc[(d_test['DIVVAL']>700)&(d_test['DIVVAL']<=11000),'DIVCAT']=1
d_train.loc[(d_train['DIVVAL']>11000)&(d_train['DIVVAL']<=28000),'DIVCAT']=2
d_test.loc[(d_test['DIVVAL']>11000)&(d_test['DIVVAL']<=28000),'DIVCAT']=2
d_train.loc[d_train['DIVVAL']>28000,'DIVCAT']=3
d_test.loc[d_test['DIVVAL']>28000,'DIVCAT']=3
d_train=d_train.drop(['DIVVAL'],axis=1)
 d_test=d_test.drop(['DIVVAL'],axis=1 )
MAX_SEQUENCE_LENGTH = 70<categorify>
numb=d_train.groupby(['FILESTAT'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb=numb.sort_values(numb.columns[2])
 numb=numb.reset_index()
le_FILESTAT = LabelEncoder()
le_FILESTAT.classes_=numb.iloc[:, 0]
d_train.loc[:, 'FILESTAT'] = le_FILESTAT.transform(d_train.loc[:, 'FILESTAT'])
 d_test.loc[:, 'FILESTAT'] = le_FILESTAT.transform(d_test.loc[:, 'FILESTAT'])
def change_word(text: str,dict_:dict)-> str:

 for s in dict_.items() :
text = text.replace(s[0],s[1])
 return text<feature_engineering>
numb=d_train.groupby(['GRINREG'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb=numb.sort_values(numb.columns[2])
 numb=numb.reset_index()
numb
d_train.loc[d_train['GRINREG']=='Not in universe','GRINREG']=1
d_test.loc[d_test['GRINREG']=='Not in universe','GRINREG']=1
d_train.loc[d_train['GRINREG']!=1,'GRINREG']=0
d_test.loc[d_test['GRINREG']!=1,'GRINREG']=0
d_train=d_train.drop(['GRINST'],axis=1)
 d_test=d_test.drop(['GRINST'],axis=1)
def lower_first_in_sentence(text: str)-> str:

 spl = re.compile('(\?+ *|\!+ *|\.+ *)')
 def lower_firts(txt): return txt if(len(txt)<2 or txt[0] =="I")else(txt[0].lower() + txt[1:])
 return ''.join([lower_firts(i)for i in re.split(spl, text)] )<categorify>
d_train.loc[d_train['HHDREL'].str.contains('Spouse of householder'),'HHDFMX']='Spouse of householder'
d_test.loc[d_test['HHDREL'].str.contains('Spouse of householder'),'HHDFMX']='Spouse of householder'
d_train.loc[(d_train['HHDREL'].str.contains('Householder')) & 
(d_train['HHDFMX'].str.contains('In group quarters')) ,'HHDFMX']='Nonfamily householder'
d_test.loc[(d_test['HHDREL'].str.contains('Householder')) & 
(d_test['HHDFMX'].str.contains('In group quarters')) ,'HHDFMX']='Nonfamily householder'
d_train.loc[d_train['HHDFMX'].str.contains('In group quarters'),'HHDFMX']='Other relative of householder'
d_test.loc[d_test['HHDFMX'].str.contains('In group quarters'),'HHDFMX']='Other relative of householder'
d_train.loc[d_train['HHDFMX'].str.contains('Other relative of householder'),'HHDREL']='Other relative of householder'
d_test.loc[d_test['HHDFMX'].str.contains('Other relative of householder'),'HHDREL']='Other relative of householder'
d_train.loc[d_train['HHDFMX'].str.contains('Grandchild'),'HHDFMX']='Child 18+'
d_test.loc[d_test['HHDFMX'].str.contains('Grandchild'),'HHDFMX']='Child 18+'
d_train.loc[d_train['HHDFMX'].str.contains('Child'),'HHDREL']='Child 18+'
d_test.loc[d_test['HHDFMX'].str.contains('Child'),'HHDREL']='Child 18+'
d_train.loc[d_train['HHDFMX'].str.contains('Child'),'HHDFMX']='Child 18+'
d_test.loc[d_test['HHDFMX'].str.contains('Child'),'HHDFMX']='Child 18+'
numb=d_train.groupby(['HHDFMX'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb=numb.reset_index()
numb=numb.sort_values(numb.columns[0])
 le_HHDFMX = LabelEncoder()
le_HHDFMX.classes_=numb.iloc[:, 0]
d_train.loc[:, 'HHDFMX'] = le_HHDFMX.transform(d_train.loc[:, 'HHDFMX'])
 d_test.loc[:, 'HHDFMX'] = le_HHDFMX.transform(d_test.loc[:, 'HHDFMX'])
 numb=d_train.groupby(['HHDREL'],)['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb=numb.reset_index()
numb=numb.sort_values(numb.columns[0])
 numb
le_HHDREL = LabelEncoder()
le_HHDREL.classes_=numb.iloc[:, 0]
d_train.loc[:, 'HHDREL'] = le_HHDREL.transform(d_train.loc[:, 'HHDREL'])
 d_test.loc[:, 'HHDREL'] = le_HHDREL.transform(d_test.loc[:, 'HHDREL'])
def strip(word:str)-> str:

 return re.sub("('$ |'$|'s |'s)",' ',word)if len(word)>2 else word<categorify>
pd.set_option('display.max_rows', 20)
 numb=d_train.groupby(['MIGMTR3'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb=numb.sort_values(numb.columns[2])
 numb=numb.reset_index()
numb
le_MIGMTR3 = LabelEncoder()
le_MIGMTR3.classes_=numb.iloc[:, 0]
d_train.loc[:, 'MIGMTR3'] = le_MIGMTR3.transform(d_train.loc[:, 'MIGMTR3'])
 d_test.loc[:, 'MIGMTR3'] = le_MIGMTR3.transform(d_test.loc[:, 'MIGMTR3'])
 d_train=d_train.drop(['MIGMTR1','MIGMTR4'],axis=1)
 d_test=d_test.drop(['MIGMTR1','MIGMTR4'],axis=1)
def clean_numbers(x):

 x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x<split>
numb=d_train.groupby(['MIGSAME'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb=numb.sort_values(numb.columns[2])
 numb=numb.reset_index()
numb
le_MIGSAME = LabelEncoder()
le_MIGSAME.classes_=numb.iloc[:, 0]
d_train.loc[:, 'MIGSAME'] = le_MIGSAME.transform(d_train.loc[:, 'MIGSAME'])
 d_test.loc[:, 'MIGSAME'] = le_MIGSAME.transform(d_test.loc[:, 'MIGSAME'] )
def load_embed(file):
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 if file == '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec':
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file)if len(o)>100)
 else:
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding='latin'))
return embeddings_index<categorify>
numb=d_train.groupby(['MIGSUN'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb=numb.sort_values(numb.columns[2])
 numb=numb.reset_index()
numb
le_MIGSUN = LabelEncoder()
le_MIGSUN.classes_=numb.iloc[:, 0]
d_train.loc[:, 'MIGSUN'] = le_MIGSUN.transform(d_train.loc[:, 'MIGSUN'])
 d_test.loc[:, 'MIGSUN'] = le_MIGSUN.transform(d_test.loc[:, 'MIGSUN'] )
def check_coverage(vocab, embeddings_index):
known_words = {}
unknown_words = {}
nb_known_words = 0
nb_unknown_words = 0
for word in vocab.keys() :
try:
known_words[word] = embeddings_index[word]
nb_known_words += vocab[word]
except:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
pass
print('Found embeddings for {:.2%} of vocab'.format(len(known_words)/ len(vocab)))
 print('Found embeddings for {:.2%} of all text'.format(nb_known_words /(nb_known_words + nb_unknown_words)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words<categorify>
d_train=d_train.drop(['PARENT'],axis=1)
 d_test=d_test.drop(['PARENT'],axis=1)
def concat_embed(first_embed, second_embed, word_index):

 WV_DIM=first_embed['I'].shape[0]+second_embed['I'].shape[0]+1
nb_words = len(word_index)+1
wv_matrix = np.zeros(shape=(nb_words, WV_DIM))
for word, i in tqdm(word_index.items()):
cap_flag = word.isupper() and word!='I'
cap = np.where(cap_flag,np.array([1]),np.array([0]))
if word in first_embed:
if word in second_embed:
wv_matrix[i] = np.hstack(( second_embed[word],first_embed[word],cap))
else:
wv_matrix[i] = np.hstack(( second_embed['something'],first_embed[word],cap))
else:
if word.lower() in first_embed:
if word.lower() in second_embed:
wv_matrix[i] = np.hstack(( second_embed[word.lower() ],first_embed[word.lower() ],cap))
else:
wv_matrix[i] = np.hstack(( second_embed['something'],first_embed[word.lower() ],cap))
else:
wv_matrix[i] = np.hstack(( second_embed['something'],first_embed['something'],cap))
return wv_matrix, WV_DIM, nb_words<categorify>
d_train=d_train.drop(['PEMNTVTY','PENATVTY'],axis=1)
 d_test=d_test.drop(['PEMNTVTY','PENATVTY'],axis=1 )
def avg_embed(first_embed, second_embed, word_index):

 WV_DIM=np.mean([second_embed['I'],first_embed['I']], axis = 0 ).shape[0]+1
nb_words = len(word_index)+1
wv_matrix = np.zeros(shape=(nb_words, WV_DIM))
for word, i in tqdm(word_index.items()):
cap_flag = word.isupper() and word!='I'
cap = np.where(cap_flag,np.array([1]),np.array([0]))
if word in first_embed:
if word in second_embed:
wv_matrix[i] = np.hstack(( np.mean([second_embed[word],first_embed[word]], axis = 0),cap))
else:
wv_matrix[i] = np.hstack(( np.mean([second_embed['something'],first_embed[word]], axis = 0),cap))
else:
if word.lower() in first_embed:
if word.lower() in second_embed:
wv_matrix[i] = np.hstack(( np.mean([second_embed[word.lower() ],first_embed[word.lower() ]], axis = 0),cap))
else:
wv_matrix[i] = np.hstack(( np.mean([second_embed['something'],first_embed[word.lower() ]], axis = 0),cap))
else:
wv_matrix[i] = np.hstack(( np.mean([second_embed['something'],first_embed['something']], axis = 0),cap))
return wv_matrix, WV_DIM, nb_words<categorify>
numb=d_train.groupby(['PEFNTVTY'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb=numb.sort_values(numb.columns[2])
 numb=numb.reset_index()
numb
le_PEFNTVTY = LabelEncoder()
le_PEFNTVTY.classes_=numb.iloc[:, 0]
d_train.loc[:, 'PEFNTVTY'] = le_PEFNTVTY.transform(d_train.loc[:, 'PEFNTVTY'])
 d_test.loc[:, 'PEFNTVTY'] = le_PEFNTVTY.transform(d_test.loc[:, 'PEFNTVTY'] )
print("Number of available cpu cores: {}".format(cpu_count()))
def process_in_parallel(function, list_):
with Pool(cpu_count())as p:
tmp = p.map(function, list_)
 return tmp<string_transform>
numb=d_train.groupby(['PRCITSHP'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb=numb.sort_values(numb.columns[2])
 numb=numb.reset_index()
numb
le_PRCITSHP = LabelEncoder()
le_PRCITSHP.classes_=numb.iloc[:, 0]
d_train.loc[:, 'PRCITSHP'] = le_PRCITSHP.transform(d_train.loc[:, 'PRCITSHP'])
 d_test.loc[:, 'PRCITSHP'] = le_PRCITSHP.transform(d_test.loc[:, 'PRCITSHP'] )
def process_questions(list_sentences, dict_, exclude):

 sleep(0.5)
 print("Lower first in sentence")
 list_sentences = process_in_parallel(lower_first_in_sentence, list_sentences)
 print("Change words - using dictionary")
 list_sentences = [change_word(s,dict_)for s in list_sentences]
print("Remove 's and solo apostrophe ' from end of the word.ex.AI's -> AI ")
 list_sentences = process_in_parallel(strip, list_sentences)
 print("Replace digits with mask ex.'Marek gets 3333' -> 'Marek gets
list_sentences = process_in_parallel(clean_numbers,list_sentences)
 print("Normalise unicode data to remove umlauts, accents etc.")
 list_sentences = [ud.normalize('NFKD', i ).encode('ASCII', 'ignore')for i in list_sentences]
print("Use TweetTokenizer() from NLTK for splitting words.")
 tokenizer = TweetTokenizer()
list_sentences = process_in_parallel(tokenizer.tokenize,list_sentences)
 print("Create vocab")
 dictionary = Counter([item for sublist in list_sentences for item in sublist])
 dictionary = OrderedDict(sorted(dictionary.items() , key=operator.itemgetter(1), reverse = True))
print("Exclude some punctuations: {}".format(" ".join(exclude)))
 list_sentences = [list(filter(lambda x: x not in exclude,i)) for i in list_sentences]
return list_sentences, dictionary<feature_engineering>
numb=d_train.groupby(['VETQVA','VETYN'])['INCCAT'].agg({'Nr':'sum','Tot':'count'})
 numb['Ratio']=numb['Nr']/numb['Tot']
numb=numb.sort_values(numb.columns[2])
 numb=numb.reset_index()
numb
le_VET = LabelEncoder()
le_VET.classes_=numb.iloc[:, 0].astype(str)+","+ numb.iloc[:, 1].astype(str)
 d_train['VET'] = le_VET.transform(d_train.loc[:, 'VETQVA'].astype(str)+ "," + d_train.loc[:,'VETYN'].astype(str))
d_test['VET'] = le_VET.transform(d_test.loc[:, 'VETQVA'].astype(str)+ "," + d_test.loc[:,'VETYN'].astype(str))
d_train=d_train.drop(['VETYN','VETQVA'],axis=1)
 d_test=d_test.drop(['VETYN','VETQVA'],axis=1)
def text_features(list_sentences, MAX_SEQUENCE_LENGTH):

 print("Column 1: Unique words rate
Column 2: Rate of all-caps words
Column 3: Sentence length rate(number of word / max sequence length parameter)")
 sleep(0.2)
 def uwr(seq): return len(set(seq)) /len(seq)if len(seq)>0 else 0
def acw(seq): return len(list(filter(lambda x: x.isupper() and x!='I', seq)))/len(seq)if len(seq)>0 else 0
def slr(seq): return len(seq)/MAX_SEQUENCE_LENGTH if len(seq)>0 else 0
uwr_f = [uwr(i)for i in tqdm(list_sentences)]
acw_f = [acw(i)for i in tqdm(list_sentences)]
slr_f = [slr(i)for i in tqdm(list_sentences)]
scaler = MinMaxScaler()
return scaler.fit_transform(np.array([uwr_f,acw_f,slr_f] ).transpose() )<define_variables>
d_train_bckp=d_train.copy()
d_test_bckp=d_test.copy()
signs = {"'":"'", "‘":"'","´": "'", "°": "","`": "'", '“': '"', '”': '"', '“': '"',
"₹": "e","€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2","—": "-",
"–": "-", "’": "'", "_": "-", "£": "e",'∞': 'infinity', 'θ': 'theta', '÷': '/',
'α': 'alpha', '•': '.', 'à': 'a', '−': '-','β': 'beta', '∅': '', '³': '3', 'π': 'pi'}
misspelled = { 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling',
'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor',
'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize',
'youtu ': 'youtube ', 'Quorans':'Quora','Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What',
'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can',
'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I',
'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation',
'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis',
'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017',
'2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex-boyfriend', 'airhostess': 'air hostess',
"whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization',
'demonitization': 'demonetization', 'demonetisation': 'demonetization',
"’":"'","‘":"'","´":"'","`":"'",'9/11':'terrorism',"Quoran":"Koran",'1/2':'half',
'cryptocurrencies':'cryptocurrency',"Brexit":'leave EU', "Blockchain":"blockchain",'.. ':''}
contractions = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because",
"could've": "could have", "couldn't": "could not", "didn't": "did not",
"doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not",
"haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is",
"how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is",
"I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have",
"I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have",
"i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have",
"isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will",
"it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam",
"mayn't": "may not", "might've": "might have","mightn't": "might not",
"mightn't've": "might not have", "must've": "must have", "mustn't": "must not",
"mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have",
"o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have",
"shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have",
"she'd": "she would", "she'd've": "she would have", "she'll": "she will",
"she'll've": "she will have", "she's": "she is", "should've": "should have",
"shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have",
"so's": "so as", "this's": "this is","that'd": "that would",
"that'd've": "that would have", "that's": "that is", "there'd": "there would",
"there'd've": "there would have", "there's": "there is", "here's": "here is",
"they'd": "they would", "they'd've": "they would have", "they'll": "they will",
"they'll've": "they will have", "they're": "they are", "they've": "they have",
"to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have",
"we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have",
"weren't": "were not", "what'll": "what will", "what'll've": "what will have",
"what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is",
"when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have",
"who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have",
"why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not",
"won't've": "will not have", "would've": "would have", "wouldn't": "would not",
"wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would",
"y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have",
"you'd": "you would", "you'd've": "you would have", "you'll": "you will",
"you'll've": "you will have", "you're": "you are", "you've": "you have" ,
"Isn't":"is not", "\u200b":"", "It's": "it is","I'm": "I am","don't":"do not"}
dict_= {}
dict_.update(signs)
 dict_.update(misspelled)
 dict_.update(contractions )<define_variables>
d_train=d_train.drop_duplicates()
exclude_list = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]<split>
add_tr=d_train[(d_train['INCCAT']==1)&(d_train['AAGE']>20)]
balanced_d_train=d_train.copy()
for i in range(0,round(balanced_d_train.shape[0]/add_tr.shape[0])) :
balanced_d_train=pd.concat([balanced_d_train,add_tr],axis=0,sort=False, ignore_index=True)
 i=i+1
sentences, vocab = process_questions(list_sentences_train+list_sentences_test, dict_, exclude_list )<feature_engineering>
gen_y_train=d_train['INCCAT'].copy()
balanced_y_train=balanced_d_train['INCCAT'].copy()
additional_features = text_features(sentences, MAX_SEQUENCE_LENGTH )<load_pretrained>
d_train = d_train.drop('INCCAT', axis=1)
 d_test = d_test.drop('INCCAT', axis=1)
 balanced_d_train=balanced_d_train.drop('INCCAT', axis=1 )
def prepare_sequences() :
word_index = {t: i+1 for i,t in enumerate(vocab)}
glove = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
wiki_news = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
print("Extracting GloVe embedding")
 embed_glove = load_embed(glove)
 print("Extracting FastText embedding")
 embed_fasttext = load_embed(wiki_news)
 print("Glove coverage: ")
 oov_glove = check_coverage(vocab, embed_glove)
 print("FastText coverage: ")
 oov_fasttext = check_coverage(vocab, embed_fasttext)
 print("Create embedding for word vocabulary")
 sleep(0.2)
 wv_matrix, WV_DIM, nb_words = concat_embed(embed_glove, embed_fasttext, word_index)
 del(globals() ['vocab'])
 gc.collect()
del(embed_glove,embed_fasttext)
 gc.collect()
sleep(5)
 print("Create sequences")
 sleep(0.2)
 sequences = [[word_index.get(t, 0)for t in sentence]
for sentence in tqdm(sentences[:len(list_sentences_train)])]
test_sequences = [[word_index.get(t, 0)for t in sentence]
for sentence in tqdm(sentences[len(list_sentences_train):])]
print("Assign additional features to train / test list")
 sleep(0.2)
 additional_features_train = additional_features[:len(list_sentences_train),:]
additional_features_test = additional_features[len(list_sentences_train):,:]
del(globals() ['list_sentences_train'])
 del(globals() ['list_sentences_test'])
 print("Pad sequences")
 sleep(0.2)
 train_data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH, padding="pre", truncating="post")
 train_df = pd.read_csv(".. /input/train.csv")
 train_y = train_df['target'].values
print('Shape of data tensor:', train_data.shape)
 print('Shape of label tensor:', train_y.shape)
 test_data = pad_sequences(test_sequences, maxlen=MAX_SEQUENCE_LENGTH, padding="pre",truncating="post")
 print('Shape of test_data tensor:', test_data.shape)
 gc.collect()
sleep(5)
 return word_index, wv_matrix, WV_DIM, nb_words, train_data, train_y, test_data, additional_features_train, additional_features_test<split>
X=balanced_d_train.copy()
y=balanced_y_train.copy()
word_index, wv_matrix, WV_DIM, nb_words, train_data, train_y, test_data, train_data_a, test_data_a = prepare_sequences()<set_options>
train_norm = X
std_scale = preprocessing.StandardScaler().fit(train_norm)
 x_train_norm = std_scale.transform(train_norm)
 training_norm_col = pd.DataFrame(x_train_norm, index=train_norm.index, columns=train_norm.columns)
 X.update(training_norm_col )
gc.collect()
sleep(10 )<define_variables>
model = Sequential()
model.add(Dense(90, input_dim=X.shape[1], activation='relu'))
model.add(Dropout(rate=0.25, name='dropout_one'))
model.add(Dense(20, activation='sigmoid'))
model.add(Dense(7, activation='sigmoid'))
model.add(Dropout(rate=0.1, name='dropout_two'))
model.add(Dense(1, activation='sigmoid'))
embed_size = WV_DIM
nb_words = nb_words
maxlen = MAX_SEQUENCE_LENGTH
batch_size = 2048
train_epochs = 6
SEED = 666
print("embed_size : {embed_size},
nb_words : {nb_words},
maxlen : {maxlen},
batch_size : {batch_size}, \
train_epochs : {train_epochs},
SEED : {SEED}".format(
 **{'embed_size':embed_size, 'nb_words':nb_words,'maxlen':maxlen,'batch_size':batch_size,'train_epochs':train_epochs,'SEED':SEED}))<import_modules>
opt=keras.optimizers.RMSprop(lr=0.003, rho=0.9)
 model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['binary_accuracy'])
import torch
import torch.nn as nn
import torch.utils.data<set_options>
np.random.seed(10)
 kfold = StratifiedKFold(n_splits=10, shuffle=True)
 cvscores = []
kfold.get_n_splits(X,y)
 i=1
for train_index, test_index in kfold.split(X, y):
print("split Nr %d" % i)
 X_tr_m1, X_te_m1 = X.iloc[train_index], X.iloc[test_index]
y_tr_m1, y_te_m1 = y.iloc[train_index], y.iloc[test_index]
model.fit(X_tr_m1, y_tr_m1, epochs=3, batch_size=6000, verbose=1)
 scores = model.evaluate(X_te_m1, y_te_m1, verbose=0)
 print("%s: %.2f%%" %(model.metrics_names[1], scores[1]*100))
cvscores.append(scores[1] * 100)
 i=i+1
print("%.2f%%(+/- %.2f%%)" %(np.mean(cvscores), np.std(cvscores)) )
def seed_torch(seed=666):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True<normalization>
opt = keras.optimizers.Adam(lr=0.03, beta_1=0.9, beta_2=0.999, amsgrad=False)
 model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['binary_accuracy'])
 model.fit(X, y, epochs=80, batch_size=8000, verbose=1 )
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )<choose_model_class>
y_T=model.predict(X)
 matrix = confusion_matrix(balanced_y_train, y_T.round())
 matrix
class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self ).__init__()
hidden_size = 64
self.embedding = nn.Embedding(nb_words, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(wv_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.1)
 self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm_attention = Attention(hidden_size*2, maxlen)
 self.gru_attention = Attention(hidden_size*2, maxlen)
 self.AvgPool1d = nn.AdaptiveAvgPool1d(1)
 self.MaxPool1d = nn.AdaptiveMaxPool1d(1)
 self.linear = nn.Linear(399, 192)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.linear2 = nn.Linear(192, 64)
 self.out = nn.Linear(64, 1)
 self.out_act = nn.Sigmoid()
def forward(self, x, x_a):
h_embedding = self.embedding(x)
 h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 h_lstm_atten = self.lstm_attention(h_lstm)
 h_gru_atten = self.gru_attention(h_gru)
 avg_pool = torch.squeeze(self.AvgPool1d(h_gru))
max_pool = torch.squeeze(self.MaxPool1d(h_gru))
conc = torch.cat(( h_gru_atten,h_lstm_atten, avg_pool, max_pool, x_a), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 conc = self.relu(self.linear2(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 y = self.out_act(out)
 return y<compute_test_metric>
roc_auc_score(balanced_y_train,y_T.round() )
def sigmoid(x):
return 1 /(1 + np.exp(-x))<compute_train_metric>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i/100 for i in range(10,90)]):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result<split>
splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED ).split(train_data, train_y))<define_variables>
X_T=d_test.copy()
test_norm = X_T
x_test_norm = std_scale.transform(test_norm)
 testing_norm_col = pd.DataFrame(x_test_norm, index=test_norm.index, columns=test_norm.columns)
 X_T.update(testing_norm_col )
warnings.filterwarnings('always')
 train_preds = np.zeros(( len(train_data)))
 test_preds = np.zeros(( len(test_data)))
 seed_torch(SEED)
 x_test_cuda = torch.tensor(test_data, dtype=torch.long ).cuda()
x_test_a_cuda = torch.tensor(test_data_a, dtype=torch.float32 ).cuda()
test_dataset = torch.utils.data.TensorDataset(x_test_cuda,x_test_a_cuda)
 test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
 loss_kFold = []
for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(train_data[train_idx], dtype=torch.long ).cuda()
x_train_a_fold = torch.tensor(train_data_a[train_idx], dtype=torch.float32 ).cuda()
y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(train_data[valid_idx], dtype=torch.long ).cuda()
x_val_a_fold = torch.tensor(train_data_a[valid_idx], dtype=torch.float32 ).cuda()
y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()
model = NeuralNet()
model.cuda()
loss_fn = torch.nn.BCELoss()
optimizer = torch.optim.Adam(model.parameters() , lr=0.001, betas=(0.9, 0.999))
train = torch.utils.data.TensorDataset(x_train_fold,x_train_a_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, x_val_a_fold,y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 loss_epoch = []
for epoch in range(train_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 losses = []
for x_batch,x_a_batch, y_batch in tqdm(train_loader, disable=True):
optimizer.zero_grad()
y_pred = model(x_batch,x_a_batch)
 loss = loss_fn(y_pred, y_batch)
 loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 losses.append(loss.data.cpu().numpy())
 loss_epoch.append(losses)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(len(test_data))
avg_val_loss = 0.
 for i,(x_batch, x_2_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch,x_2_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = y_pred.cpu().numpy() [:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, train_epochs, avg_loss, avg_val_loss, elapsed_time))
for i,(x_batch,x_a_batch)in enumerate(test_loader):
y_pred = model(x_batch,x_a_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = y_pred.cpu().numpy() [:, 0]
train_preds[valid_idx] = valid_preds_fold
print(threshold_search(train_y[valid_idx], train_preds[valid_idx]))
test_preds += test_preds_fold / len(splits)
 loss_kFold.append(loss_epoch )<compute_test_metric>
y_Test_lev1_pred=model.predict(X_T )
search_result = threshold_search(train_y, train_preds)
 search_result<save_to_csv>
d_test['y_new']=y_Test_lev1_pred.round()
sum(d_test['y_new'] )
test_df = pd.read_csv(".. /input/test.csv")
 sub = pd.DataFrame({"qid": test_df["qid"].values})
 sub['prediction'] =(test_preds > search_result['threshold'] ).astype(int)
 sub.to_csv("submission.csv", index=False )<define_variables>
d_test.loc[d_test['AAGE']<21,['y_new']]=0
sum(d_test['y_new'] )
print("The percentage of insincere questions in train set is {:.2%}.".format(train_1_prc))<define_variables>
sub=pd.read_csv('/kaggle/input/ml-challenge-week6/sampleSubmission.csv')
 sub["income class"]=d_test["y_new"]
<string_transform><EOS> 
filename = 'submission.csv'
sub.to_csv(filename,index=False )