Kaggle competition: Titanic - Machine Learning from Disaster
if USE_LGB:
word_categories = ['adjectives','verbs','nouns','list_maker','digits','modals','posessives','persionals','interjection','direction','past_verb']
adjectives = ['JJ','JJR','JJS','RB','RBR','RBS']
verbs = ['VB','VBD','VBG','VBN','VBP','VBZ']
nouns = ['NN','NNS','NNP','NNPS']
list_maker = ['LS']
digits = ['CD']
modals = ['MD']
posessives = ['PPR$', 'POS']
persionals = ['PRP']
interjection = ['UH']
direction = ['TO']
past_verb = ['VBD','VBN']
def get_string_stats(df, var):
df.loc[:, f'{var}_numchars'] = df[var].apply(lambda x: len(x))
df.loc[:, f'{var}_numwords'] = df[var].apply(lambda x: len(x.split()))
df.loc[:, f'{var}_exclam_count'] = df[var].apply(lambda x: x.count('!'))
df.loc[:, f'{var}_question_count'] = df[var].apply(lambda x: x.count('?'))
df.loc[:, f'{var}_coma_count'] = df[var].apply(lambda x: x.count(','))
df.loc[:, f'{var}_dot_count'] = df[var].apply(lambda x: x.count('.'))
df.loc[:, f'{var}_all_punct_count'] = df[f'{var}_question_count']+df[f'{var}_coma_count']+df[f'{var}_exclam_count']
df.loc[:, f'{var}_all_punct_to_sentences'] = df.loc[:, f'{var}_all_punct_count']/df.loc[:, f'{var}_dot_count']
df.loc[:, f'{var}_questions_to_sentences'] = df.loc[:, f'{var}_question_count']/df.loc[:, f'{var}_dot_count']
df.loc[:, f'{var}_questions_to_words'] = df.loc[:, f'{var}_question_count']/df.loc[:, f'{var}_numwords']
df.loc[:, f'{var}_average_word_len'] = df[f'{var}_numchars']/df[f'{var}_numwords']
df.loc[:, f'{var}_capital_count'] = df[var].apply(lambda x: sum(1 for c in x if c.isupper()))
df.loc[:, f'{var}_capital_prop'] = df[f'{var}_capital_count']/df[f'{var}_numwords']
df.loc[:, f'{var}_other_ref'] = df[var].apply(lambda x: sum([x.count(i)for i in [' it ',' they ', " it's ", ' their ']]))
df.loc[:, f'{var}_self_ref'] = df[var].apply(lambda x: sum([x.count(i)for i in [' I ',' me ', " mine ", ' my ']]))
df.loc[:, f'{var}_total_ref'] = df[f'{var}_self_ref'] + df[f'{var}_other_ref']
df.loc[:, f'{var}_total_ref_prop'] = df.loc[:, f'{var}_total_ref']/df.loc[:, f'{var}_numwords']
df.loc[:, f'{var}_self_ref_prop'] = df[f'{var}_self_ref']/df[f'{var}_total_ref']
df.loc[:, f'{var}_other_ref_prop'] = df[f'{var}_other_ref']/df[f'{var}_total_ref']
df.loc[:, f'{var}_words_per_sentence'] = df[f'{var}_numwords']/df[f'{var}_dot_count']
df.loc[:, f'{var}_unique_words'] = df[f'{var}'].apply(lambda x: len(set(str(x ).split())))
 df.loc[:, f'{var}_unique_words_prop'] = df.loc[:, f'{var}_unique_words']/df.loc[:, f'{var}_numwords']
new_cols = [f'{var}_total_ref_prop', f'{var}_questions_to_words',f'{var}_questions_to_sentences',f'{var}_all_punct_to_sentences', f'{var}_unique_words_prop', f'{var}_unique_words', f'{var}_numchars',f'{var}_numwords',f'{var}_exclam_count',f'{var}_question_count',f'{var}_coma_count',f'{var}_dot_count',f'{var}_all_punct_count',f'{var}_average_word_len',f'{var}_capital_count',f'{var}_capital_prop',f'{var}_other_ref',f'{var}_self_ref',f'{var}_total_ref',f'{var}_self_ref_prop',f'{var}_other_ref_prop',f'{var}_words_per_sentence']
for category in word_categories:
df.loc[:, f'{var}_{category}'] = 0
new_cols.append(f'{var}_{category}')
 for idx in tqdm(range(df.shape[0]), total = df.shape[0]):
tokens = nltk.word_tokenize(df.loc[idx, var])
 tags = nltk.pos_tag(tokens)
 tags = [i[1] for i in tags]
for category in word_categories:
count = 0
for tag_name in globals() [category]:
count += tags.count(tag_name)
 df.loc[idx, f'{var}_{category}'] = count/df.loc[idx, f'{var}_numwords'] if df.loc[idx, f'{var}_numwords'] != 0 else 0
return df, new_cols
def get_extra_features_and_map(df):
df, nc1 = get_string_stats(df, 'question_title')
 df, nc2 = get_string_stats(df, 'question_body')
 df, nc3 = get_string_stats(df, 'answer')
 df.loc[:,'q_user_q_count'] = df.q_users_host.apply(lambda x: q_users[x] if x in q_users else 0)
 df.loc[:,'q_user_a_count'] = df.q_users_host.apply(lambda x: a_users[x] if x in a_users else 0)
 df.loc[:,'a_user_a_count'] = df.a_users_host.apply(lambda x: a_users[x] if x in a_users else 0)
 df.loc[:,'a_user_q_count'] = df.a_users_host.apply(lambda x: q_users[x] if x in q_users else 0)
 df.loc[:,'q_user_both_count'] = df.loc[:,'q_user_q_count'] + df.loc[:,'q_user_a_count']
df.loc[:,'a_user_both_count'] = df.loc[:,'a_user_a_count'] + df.loc[:,'a_user_q_count']
other_features = []
df.loc[:,'q_to_a_all_punct_count'] =(df.loc[:,'question_body_all_punct_count']+df.loc[:,'question_title_all_punct_count'])/df.loc[:,'answer_all_punct_count']
df.loc[:,'q_to_a_numwords'] =(df.loc[:,'question_body_numwords']+df.loc[:,'question_title_numwords'])/df.loc[:,'answer_numwords']
df.loc[:,'q_to_a_capital_count'] =(df.loc[:,'question_body_capital_count']+df.loc[:,'question_title_capital_count'])/df.loc[:,'answer_capital_count']
df.loc[:,'q_to_a_unique_words_prop'] =(df.loc[:,'question_body_unique_words_prop']+df.loc[:,'question_title_unique_words_prop'])/df.loc[:,'answer_unique_words_prop']
df.loc[:,'q_to_a_total_ref'] =(df.loc[:,'question_body_total_ref']+df.loc[:,'question_title_total_ref'])/df.loc[:,'answer_total_ref']
df.loc[:,'common_words'] = df.apply(lambda x: len(set(x.question_body.split() ).intersection(set(x.answer.split()))), axis=1)
 other_features += ['q_to_a_all_punct_count', 'q_to_a_numwords', 'q_to_a_capital_count', 'common_words', 'q_to_a_unique_words_prop', 'q_to_a_total_ref']
for category in word_categories:
df.loc[:,f'q_to_a_{category}'] = df.loc[:,f'question_body_{category}']/df.loc[:,f'answer_{category}']
other_features.append(f'q_to_a_{category}')
 df.loc[:,'spell_words'] = df.loc[:,'question_title'].apply(lambda x: sum(1 for i in x.lower().split() if i in ['spell','spelled','spelt','spelling','write','wrote','written']))
df.loc[:,'spell_words'] += df.loc[:,'question_body'].apply(lambda x: sum(1 for i in x.lower().split() if i in ['spell','spelled','spelt','spelling','write','wrote','written']))
df.loc[:,'compare_words'] = df.loc[:,'question_title'].apply(lambda x: sum(1 for i in x.lower().split() if i in ['better','best','worse','nicer']))
df.loc[:,'compare_words'] += df.loc[:,'question_body'].apply(lambda x: sum(1 for i in x.lower().split() if i in ['better','best','worse','nicer']))
df.loc[:,'consequence_words'] = df.loc[:,'question_title'].apply(lambda x: sum(1 for i in x.lower().split() if i in ['if','when','will','would']))
df.loc[:,'consequence_words'] += df.loc[:,'question_body'].apply(lambda x: sum(1 for i in x.lower().split() if i in ['if','when','will','would']))
other_features.append('spell_words')
 other_features.append('compare_words')
 other_features.append('consequence_words')
 onehots=[]
for i in df.loc[:,'category'].unique() :
df.loc[:,f'{i}_onehot'] = 0
df.loc[df.loc[:,'category']==i,f'{i}_onehot'] = 1
onehots.append(f'{i}_onehot')
 for i in df.loc[:,'host'].unique() :
df.loc[:,f'{i}_H_onehot'] = 0
df.loc[df.loc[:,'host']==i,f'{i}_H_onehot'] = 1
onehots.append(f'{i}_H_onehot')
 other_features = other_features+nc1+nc2+nc3+onehots+['q_user_q_count', 'q_user_a_count','a_user_a_count','a_user_q_count','q_user_both_count','a_user_both_count']
return df, other_features
df_train, other_features_train = get_extra_features_and_map(df_train)
 df_test, other_features = get_extra_features_and_map(df_test)
 for i in [a for a in other_features if a not in other_features_train]:
df_train.loc[:,i] = np.zeros(df_train.shape[0] )<feature_engineering>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()
if USE_LGB:
def get_uids_all(df):
df.loc[:,'answer_uid'] = df.loc[:,'answer_user_page'].apply(lambda x: int(x.split('/')[-1]))
df.loc[:,'question_uid'] = df.loc[:,'question_user_page'].apply(lambda x: int(x.split('/')[-1]))
for idx in range(df.shape[0]):
split = [i for i in df.loc[idx,'url'].split('/')if i.isdigit() ]
df.loc[idx,'url_uid'] = int(split[-1])if len(split)>0 else -1
return df
df_train = get_uids_all(df_train)
 df_test= get_uids_all(df_test )<load_from_csv>
train.isnull().sum()
if USE_LGB:
se_path = ".. /input/stackexchange-data"
se_posts = pd.read_parquet(se_path+"/stackexchange_posts.parquet.gzip", engine='fastparquet')
 def get_post_info_se(df):
new_other_features = []
new_features = ['Score','ViewCount','AnswerCount','CommentCount','FavoriteCount','Tags']
df = df.merge(se_posts.loc[:,['Id', 'host', 'AcceptedAnswerId'] + new_features], how='left', left_on=['url_uid', 'host'], right_on=['Id', 'host'], sort=False)
 df.rename({i:'SE_QP_'+i for i in new_features}, inplace=True, axis=1)
 new_other_features += ['SE_QP_'+i for i in new_features]
return df, new_other_features
df_train, new_other_features = get_post_info_se(df_train)
 df_test, _ = get_post_info_se(df_test)
 del se_posts
gc.collect()
other_features += new_other_features
other_features.remove('SE_QP_Tags')
 all_tags = []
for i in range(df_train.shape[0]):
if(df_train.SE_QP_Tags.iloc[i] == None)or(pd.isna(df_train.SE_QP_Tags.iloc[i])) : continue
tags = df_train.SE_QP_Tags.iloc[i].replace('<','' ).split('>')
 all_tags += [t for t in tags if len(t)>0]
top_tags = list(pd.DataFrame(all_tags ).iloc[:,0].value_counts() [0:50].index)
 other_features += [f'tag_{i}' for i in top_tags]
for t in top_tags:
df_train.loc[:,f'tag_{t}'] = 0
df_test.loc[:,f'tag_{t}'] = 0
def parse_tags(df):
for i in range(df.shape[0]):
if(df.SE_QP_Tags.iloc[i] == None)or(pd.isna(df.SE_QP_Tags.iloc[i])) : continue
tags = df.SE_QP_Tags.iloc[i].replace('<','' ).split('>')
 tags = [t for t in tags if(len(t)> 0)&(t in top_tags)]
for t in tags:
df.loc[i,f'tag_{t}'] = 1
return df
df_train = parse_tags(df_train)
 df_test = parse_tags(df_test)
 se_path = ".. /input/stackexchange-data"
se_users = pd.read_parquet(se_path+"/stackexchange_users.parquet.gzip", engine='fastparquet')
 def get_user_info_se(df):
new_other_features = []
new_features = ['Reputation','Views','Upvotes','Downvotes']
df = df.merge(se_users.loc[:,['Id', 'host'] + new_features], how='left', left_on=['question_uid', 'host'], right_on=['Id', 'host'], sort=False)
 df.rename({i:'SE_Q_'+i for i in new_features}, inplace=True, axis=1)
 new_other_features += ['SE_Q_'+i for i in new_features]
new_features = ['Reputation','Views','Upvotes','Downvotes']
df = df.merge(se_users.loc[:,['Id', 'host'] + new_features], how='left', left_on=['answer_uid', 'host'], right_on=['Id', 'host'], sort=False)
 df.rename({i:'SE_A_'+i for i in new_features}, inplace=True, axis=1)
 new_other_features += ['SE_A_'+i for i in new_features]
return df, new_other_features
df_train, new_other_features = get_user_info_se(df_train)
 df_test, _ = get_user_info_se(df_test)
 del se_users
gc.collect()
other_features += new_other_features<feature_engineering>
test.isnull().sum()
if USE_LGB:
def is_answer_accepted(df):
for i in df.loc[(df.host=='stackoverflow.com')& ~(df.QPAGE_accepted_answer_id.isna()),:].index.values:
df.loc[i,'answer_accepted'] = 1 if df.loc[i, 'answer_uid'] == df.loc[i, 'QPAGE_accepted_answer_id'] else 0
for i in df.loc[(df.host!='stackoverflow.com')& ~(df.AcceptedAnswerId.isna()),:].index.values:
df.loc[i,'answer_accepted'] = 1 if df.loc[i, 'answer_uid'] == df.loc[i, 'AcceptedAnswerId'] else 0
return df
df_train = is_answer_accepted(df_train)
 df_test = is_answer_accepted(df_test)
 other_features.append('answer_accepted' )<train_on_grid>
test.isnull().sum()
if USE_LGB:
class Base_Model(object):
def __init__(self, train_df, test_df, features, categoricals=[], n_splits=5, verbose=True, target=None, predict_test=True):
self.train_df = train_df
self.test_df = test_df
self.features = features
self.n_splits = 10
self.categoricals = categoricals
self.target = target
self.cv = self.get_cv()
self.verbose = verbose
self.params = self.get_params()
self.predict_test = predict_test
self.tst_pred, self.score, self.model, self.val_ys, self.val_preds = self.fit()
def train_model(self, train_set, val_set):
raise NotImplementedError
def get_cv(self):
cv = GroupKFold(n_splits=self.n_splits)
 return cv.split(X=self.train_df.question_body_orig, groups=self.train_df.question_body_orig)
 def get_params(self):
raise NotImplementedError
def convert_dataset(self, x_train, y_train, x_val, y_val):
raise NotImplementedError
def convert_x(self, x):
return x
def fit(self):
oof_pred = []
oof_ys_all = []
y_pred = np.zeros(( len(self.test_df),))
for fold,(train_idx, val_idx)in enumerate(self.cv):
if fold < ACTUAL_FOLDS:
x_train, x_val = self.train_df[self.features].iloc[train_idx], self.train_df[self.features].iloc[val_idx]
y_train, y_val = self.train_df[self.target][train_idx], self.train_df[self.target][val_idx]
train_set, val_set = self.convert_dataset(x_train, y_train, x_val, y_val)
 model = self.load_model(fold)
 conv_x_val = self.convert_x(x_val.reset_index(drop=True))
preds_all = model.predict(conv_x_val)
 preds_all = 1/(1 + np.exp(-preds_all))
oof_pred += preds_all.tolist()
if self.predict_test:
x_test = self.convert_x(self.test_df[self.features])
 current_test_preds = model.predict(x_test ).reshape(y_pred.shape)
 current_test_preds = 1/(1 + np.exp(-current_test_preds))
current_test_preds = current_test_preds/ACTUAL_FOLDS
y_pred += current_test_preds
if self.verbose:print('Partial score(all)of fold {} is: {}'.format(fold, spearmanr(y_val, preds_all ).correlation))
oof_ys_all += list(y_val.reset_index(drop=True ).values)
 loss_score = spearmanr(oof_ys_all, oof_pred ).correlation
if self.verbose:
print('Our oof cohen kappa score(all)is: ', loss_score)
 return y_pred, loss_score, model, np.array(oof_ys_all), np.array(oof_pred)
 class Lgb_Model(Base_Model):
def train_model(self, train_set, val_set):
verbosity = 100 if self.verbose else 0
return lgb.train(self.params, train_set, valid_sets=[train_set, val_set], verbose_eval=verbosity)
 def load_model(self, fold):
model = pickle.load(open(f'.. /input/gq-lgb/{self.target}/{self.target}_{fold}.p', 'rb'))
return model
def convert_dataset(self, x_train, y_train, x_val, y_val):
train_set = lgb.Dataset(x_train, y_train, categorical_feature=self.categoricals)
 val_set = lgb.Dataset(x_val, y_val, categorical_feature=self.categoricals)
 return train_set, val_set
def get_params(self):
params = {'n_estimators':5000,
'boosting_type': 'gbdt',
'objective': 'cross_entropy_lambda',
'subsample': 0.75,
'subsample_freq': 1,
'learning_rate': 0.01,
'feature_fraction': 0.8,
'max_depth': 150,
'num_leaves': 50,
'lambda_l1': 0.1,
'lambda_l2': 0.1,
'early_stopping_rounds': 300,
'min_data_in_leaf': 1,
'min_gain_to_split': 0.01,
'max_bin': 400
}
return params
class Xgb_Model(Base_Model):
def load_model(self, fold):
model = pickle.load(open(f'.. /input/gq-xgb/{self.target}/{self.target}_{fold}.p', 'rb'))
return model
def train_model(self, train_set, val_set):
verbosity = 100 if self.verbose else 0
return xgb.train(self.params, train_set,
num_boost_round=5000, evals=[(train_set, 'train'),(val_set, 'val')],
verbose_eval=verbosity, early_stopping_rounds=100)
 def convert_dataset(self, x_train, y_train, x_val, y_val):
train_set = xgb.DMatrix(x_train, y_train)
 val_set = xgb.DMatrix(x_val, y_val)
 return train_set, val_set
def convert_x(self, x):
return xgb.DMatrix(x)
 def get_params(self):
params = {
'objective':'reg:logistic',
'n_estimators':5000,
'max_depth':12,
'eta':0.05}
return params<feature_engineering>
train_test_data = [train, test]
if USE_LGB:
one_lgb_model = pickle.load(open(f'.. /input/gq-lgb/question_opinion_seeking/question_opinion_seeking_0.p', 'rb'))
lgb_pretrained_features = one_lgb_model.feature_name()
for i in lgb_pretrained_features:
if i not in other_features:
print(f'{i} not in other features here, adding zeros')
 df_train.loc[:, i] = np.zeros(df_train.shape[0])
 df_test.loc[:, i] = np.zeros(df_test.shape[0] )<choose_model_class>
train.groupby('Pclass' ).mean()
%%time
if USE_LGB:
ACTUAL_FOLDS = 3
lgb_val_scores = []
n_output_categories = len(output_categories)
 lgb_val_outputs_all = []
lgb_val_preds_all = []
lgb_tst_preds_all = []
for idx, i in enumerate(output_categories, 1):
lgb_model = Lgb_Model(df_train, df_test, lgb_pretrained_features, target=i, verbose=False)
 lgb_val_outputs_all.append(lgb_model.val_ys)
 lgb_val_preds_all.append(lgb_model.val_preds)
 lgb_tst_preds_all.append(lgb_model.tst_pred)
 lgb_val_scores.append(lgb_model.score)
 print(f'{idx}/{n_output_categories}',i, lgb_model.score)
 save_file(lgb_val_outputs_all, 'lgb_val_outputs_all')
 save_file(lgb_val_preds_all, 'lgb_val_preds_all')
 save_file(lgb_tst_preds_all, 'lgb_tst_preds_all')
 save_file(df_train, 'df_train')
 save_file(df_test, 'df_test')
 else:
lgb_val_outputs_all = [0]*30
lgb_val_preds_all = [0]*30
lgb_tst_preds_all = [0]*30<feature_engineering>
encoder=LabelBinarizer()
train['Sex']=encoder.fit_transform(train['Sex'])
 test['Sex']=encoder.fit_transform(test['Sex'])
 train.head(10)
if USE_XGB:
one_xgb_model = pickle.load(open(f'.. /input/gq-xgb/question_opinion_seeking/question_opinion_seeking_0.p', 'rb'))
xgb_pretrained_features = one_xgb_model.feature_names
for i in xgb_pretrained_features:
if i not in other_features:
print(f'{i} not in other features here, adding zeros')
 df_train.loc[:, i] = np.zeros(df_train.shape[0])
 df_test.loc[:, i] = np.zeros(df_test.shape[0] )<choose_model_class>
for dataset in train_test_data:
dataset['Name'] = dataset['Name'].str.extract('([A-Za-z]+)\.', expand=True )
%%time
if USE_XGB:
ACTUAL_FOLDS = 3
xgb_val_scores = []
n_output_categories = len(output_categories)
 xgb_val_outputs_all = []
xgb_val_preds_all = []
xgb_tst_preds_all = []
for idx, i in enumerate(output_categories, 1):
xgb_model = Xgb_Model(df_train, df_test, xgb_pretrained_features, target=i, verbose=False)
 xgb_val_outputs_all.append(xgb_model.val_ys)
 xgb_val_preds_all.append(xgb_model.val_preds)
 xgb_tst_preds_all.append(xgb_model.tst_pred)
 xgb_val_scores.append(xgb_model.score)
 print(f'{idx}/{n_output_categories}',i, xgb_model.score)
 save_file(xgb_val_outputs_all, 'xgb_val_outputs_all')
 save_file(xgb_val_preds_all, 'xgb_val_preds_all')
 save_file(xgb_tst_preds_all, 'xgb_tst_preds_all')
 save_file(df_train, 'df_train')
 save_file(df_test, 'df_test')
 else:
xgb_val_outputs_all = [0]*30
xgb_val_preds_all = [0]*30
xgb_tst_preds_all = [0]*30<compute_test_metric>
train['Name'].value_counts()
def get_rounding(ys, preds):
rounding_types = ['Normal', 'Ceil', 'Floor']
rounding_funcs = [np.round, np.ceil, np.floor]
dec_places = [1,2,3,4,5]
score = spearmanr(ys, preds ).correlation
if np.isnan(score): score=-100
best_result = {'Type':'No rounding','DP':0, 'func':None}
for r_type, r_func in zip(rounding_types, rounding_funcs):
for dp in dec_places:
if r_type == 'Normal':
rounded_preds = r_func(preds, dp)
 cur_score = spearmanr(ys, rounded_preds ).correlation
else:
rounded_preds = r_func(preds*(10**dp)) /(10**dp)
 cur_score = spearmanr(ys, rounded_preds ).correlation
if np.isnan(cur_score): cur_score = 0
if cur_score > score:
score = cur_score
best_result['Type'] = r_type
best_result['DP'] = dp
best_result['func'] = r_func
return score, best_result<compute_train_metric>
replace_name = {'Mlle': 'Miss', 'Major': 'Mr', 'Col': 'Mr', 'Sir': 'Mr', 'Don': 'Mr', 'Mme': 'Miss',
'Jonkheer': 'Mr', 'Lady': 'Mrs', 'Capt': 'Mr', 'Countess': 'Mrs', 'Ms': 'Miss', 'Dona': 'Mrs'}
def inverse_spearman_r(weights, ys, preds):
mixed_val_preds = np.array([i*w for i,w in zip(preds, weights)] ).sum(axis=0)
 score = spearmanr(ys, mixed_val_preds ).correlation
if np.isnan(score): score=-100
return -score
def optimize_mixing_weights(ys, preds):
naive_mix = np.array(preds ).mean(axis=0)
 score = spearmanr(ys, naive_mix ).correlation
if np.isnan(score): score=-100
c_dict = {'type':'eq','fun':lambda x: 1-np.sum(x)}
optim_func = partial(inverse_spearman_r, ys=ys, preds=preds)
 x0 = np.array([1/len(preds)]*len(preds))
res = minimize(optim_func, x0, method='SLSQP', constraints=c_dict)
 print(f'Best score {res.fun}; weights {res.x}')
 return res.x
def optimize_mixing_weights_guessing(ys, preds):
variants = [0.0, 0.01, 0.04, 0.1, 0.12, 0.16, 0.18, 0.2, 0.22, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.82, 0.85, 0.88, 0.9, 0.94, 0.95]
models_n = len(preds)
 weights = [1/models_n]*models_n
best_weights = weights
best_score = 0
for comb in combinations_with_replacement(variants, models_n):
weights = list(comb)
 if np.sum(weights)!= 1: continue
score = spearmanr(ys, np.array([i*w for i,w in zip(preds, weights)] ).sum(axis=0)).correlation
if np.isnan(score): score=-100
if score>best_score: best_weights, best_score = weights.copy() , score
print(f'Best score {best_score}; weights {best_weights}')
 return best_weights<define_variables>
train.replace({'Name' : replace_name}, inplace=True )
val10_common = [np.where(val10==i)[0][0] for i in common_validation_idx]
val5_common = [np.where(val5==i)[0][0] for i in common_validation_idx]<define_variables>
test.replace({'Name' : replace_name}, inplace=True )
val10_fold0_common = [np.where(val10_fold0==i)[0] for i in common_validation_idx]
val10_fold0_common = [i[0] for i in val10_fold0_common if len(i)>0]<load_from_csv>
train[['Name', 'Survived']].groupby(['Name'], as_index=False ).mean()
%%time
weights_all = []
weights_for_rounding = []
scores_all = []
df_sub = pd.read_csv(PATH+'sample_submission.csv')
 bestonly_raw_scores = []
best_rounded_scores = []
best_weighted_scores = []
if USE_SAMPLE: df_sub=df_sub.iloc[0:round(0.1*df_sub.shape[0]),:]
if USE_LGB:
if len(lgb_val_scores)!= 30:
USE_LGB=False
print('Something wrong with LGB, switching it off')
 for idx, cat in enumerate(output_categories):
lgb_score = spearmanr(xlnet_val[1][:,idx], lgb_val_preds_all[idx] ).correlation if USE_LGB else np.nan
xgb_score = spearmanr(xlnet_val[1][:,idx], xgb_val_preds_all[idx] ).correlation if USE_XGB else np.nan
gpt2_score = spearmanr(gpt2_val[1][:,idx], gpt2_val[0][:,idx] ).correlation
bert_score = spearmanr(bert_val[1][:,idx], bert_val[0][:,idx] ).correlation
xlnet_score = spearmanr(xlnet_val[1][:,idx], xlnet_val[0][:,idx] ).correlation
bert_score_3split = spearmanr(bert_val_3split[1][:,idx], bert_val_3split[0][:,idx] ).correlation if USE_3SPLIT_BERT else np.nan
bert_score_5fold = spearmanr(bert_val_5fold[1][:,idx], bert_val_5fold[0][:,idx] ).correlation
gpt2_score_5fold = spearmanr(gpt2_val_5fold[1][:,idx], gpt2_val_5fold[0][:,idx] ).correlation
scores_all.append([lgb_score, xgb_score, gpt2_score, bert_score, xlnet_score, bert_score_3split, bert_score_5fold, gpt2_score_5fold])
 selected_models_scores = [i for i in [lgb_score, xgb_score, gpt2_score, bert_score, xlnet_score, bert_score_3split, bert_score_5fold, gpt2_score_5fold] if not np.isnan(i)]
selected_models_val_preds = [i for i, s in zip([lgb_val_preds_all[idx], xgb_val_preds_all[idx], gpt2_val[0][:,idx], bert_val[0][:,idx], xlnet_val[0][:,idx], bert_val_3split[0][:,idx], bert_val_5fold[0][:,idx], gpt2_val_5fold[0][:,idx]],[lgb_score, xgb_score, gpt2_score, bert_score, xlnet_score, bert_score_3split, bert_score_5fold, gpt2_score_5fold])if not np.isnan(s)]
selected_models_tst_preds = [i for i, s in zip([lgb_tst_preds_all[idx], xgb_tst_preds_all[idx], gpt2_tst[:,idx], bert_tst[:,idx], xlnet_tst[:,idx], bert_tst_3split[:,idx], bert_tst_5fold[:,idx], gpt2_tst_5fold[:,idx]],[lgb_score, xgb_score, gpt2_score, bert_score, xlnet_score, bert_score_3split, bert_score_5fold, gpt2_score_5fold])if not np.isnan(s)]
current_best = np.max(selected_models_scores)
 bestonly_raw_scores.append(current_best)
 if USE_LGB & USE_XGB:
val_preds_10and5 = [lgb_val_preds_all[idx][val10_common], xgb_val_preds_all[idx][val10_common], gpt2_val[0][val10_common,idx], bert_val[0][val10_common,idx], xlnet_val[0][val10_common,idx], bert_val_5fold[0][val5_common,idx], gpt2_val_5fold[0][val5_common,idx]]
scores10and5 = np.sum([np.exp(i*beta)for i in [lgb_score, xgb_score, gpt2_score, bert_score, xlnet_score, bert_score_5fold, gpt2_score_5fold]])
 weights10and5 = [np.exp(i*beta)/scores10and5 for i in [lgb_score, xgb_score, gpt2_score, bert_score, xlnet_score, bert_score_5fold, gpt2_score_5fold]]
if USE_LGB &(not USE_XGB):
val_preds_10and5 = [lgb_val_preds_all[idx][val10_common], gpt2_val[0][val10_common,idx], bert_val[0][val10_common,idx], xlnet_val[0][val10_common,idx], bert_val_5fold[0][val5_common,idx], gpt2_val_5fold[0][val5_common,idx]]
scores10and5 = np.sum([np.exp(i*beta)for i in [lgb_score, gpt2_score, bert_score, xlnet_score, bert_score_5fold, gpt2_score_5fold]])
 weights10and5 = [np.exp(i*beta)/scores10and5 for i in [lgb_score, gpt2_score, bert_score, xlnet_score, bert_score_5fold, gpt2_score_5fold]]
if(not USE_LGB)&(not USE_XGB):
val_preds_10and5 = [gpt2_val[0][val10_common,idx], bert_val[0][val10_common,idx], xlnet_val[0][val10_common,idx], bert_val_5fold[0][val5_common,idx], gpt2_val_5fold[0][val5_common,idx]]
scores10and5 = np.sum([np.exp(i*beta)for i in [gpt2_score, bert_score, xlnet_score, bert_score_5fold, gpt2_score_5fold]])
 weights10and5 = [np.exp(i*beta)/scores10and5 for i in [gpt2_score, bert_score, xlnet_score, bert_score_5fold, gpt2_score_5fold]]
val_mix = np.array([i*w for i,w in zip(val_preds_10and5, weights10and5)] ).sum(axis=0)
 common_val = xlnet_val[1][val10_common,idx]
weights_for_rounding.append(weights10and5)
 if not ROUND: print(f'{idx} {cat} scores: LGB: {lgb_score}, GPT2: {gpt2_score}, BERT: {bert_score}, XLNET: {xlnet_score}')
 if WEIGHTS == 'softmax':
all_scores = np.sum(np.exp([i*beta for i in selected_models_scores]))
weights = [np.exp(i*beta)/all_scores for i in selected_models_scores]
best_weighted_scores.append(spearmanr(common_val, np.array([i*w for i,w in zip(val_preds_10and5, weights)] ).sum(axis=0)).correlation)
 weights_all.append(weights)
 df_sub.iloc[:, idx+1] = np.array([i*w for i,w in zip(selected_models_tst_preds, weights)] ).sum(axis=0)
 if WEIGHTS == 'dynamic':
weights = optimize_mixing_weights_guessing(common_val, val_preds_10and5)
 weights_all.append(weights)
 best_weighted_scores.append(spearmanr(common_val, np.array([i*w for i,w in zip(val_preds_10and5, weights)] ).sum(axis=0)).correlation)
 val_mix = np.array([i*w for i,w in zip(val_preds_10and5, weights)] ).sum(axis=0)
 df_sub.iloc[:, idx+1] = np.array([i*w for i,w in zip(selected_models_tst_preds, weights)] ).sum(axis=0)
 if WEIGHTS == 'dynamic softmax':
ds_score = -100
best_beta = None
for b in np.arange(1,30,0.5):
all_scores = np.sum(np.exp([i*b for i in selected_models_scores]))
try_weights = [np.exp(i*b)/all_scores for i in selected_models_scores]
try_mix = np.array([i*w for i,w in zip(val_preds_10and5, try_weights)] ).sum(axis=0)
 try_score = spearmanr(common_val, try_mix ).correlation
if try_score > ds_score:
best_beta = b
ds_score = try_score
weights = try_weights.copy()
val_mix = try_mix.copy()
weights_all.append(weights)
 best_weighted_scores.append(ds_score)
 print(f'Best beta: {best_beta}')
 df_sub.iloc[:, idx+1] = np.array([i*w for i,w in zip(selected_models_tst_preds, weights)] ).sum(axis=0)
 if WEIGHTS == 'bestonly':
df_sub.iloc[:, idx+1] = selected_models_tst_preds[np.argmax(selected_models_scores)]
if ROUND:
best_rounded_score, rounding_method = get_rounding(common_val, val_mix)
 best_rounded_scores.append(best_rounded_score)
 print(f"{idx} {cat}: mixed score {best_rounded_score}; {rounding_method['Type']}, {rounding_method['DP']}")
 unrounded_backup = df_sub.iloc[:, idx+1].copy()
if rounding_method['Type'] == 'Normal':
df_sub.iloc[:, idx+1] = rounding_method['func'](df_sub.iloc[:, idx+1], rounding_method['DP'])
 elif(rounding_method['Type'] == 'Ceil')or(rounding_method['Type'] == 'Floor'):
df_sub.iloc[:, idx+1] = rounding_method['func'](df_sub.iloc[:, idx+1]*(10**rounding_method['DP'])) /(10**rounding_method['DP'])
 if USE_TEST_SAFETY_ADJUSTMENTS:
if df_sub.iloc[:,idx+1].var() == 0:
print('Test predictions are STILL homogenous, reverting to softmax weights')
 all_scores = np.sum(np.exp([i for i in selected_models_scores]))
softmax_weights = [np.exp(i)/all_scores for i in selected_models_scores]
df_sub.iloc[:, idx+1] = np.array([i*w for i,w in zip(selected_models_tst_preds, softmax_weights)] ).sum(axis=0)
 if df_sub.iloc[:,idx+1].var() == 0:
raise Exception('var = 0')
 if df_sub.iloc[:,idx+1].min() < 0:
raise Exception("<0")
 if df_sub.iloc[:,idx+1].max() > 1:
raise Exception(">1")
 if df_sub.isna().sum().sum() != 0:
raise Exception("na in sub")
 df_sub.to_csv('submission.csv', index=False )<save_model>
score_name = {"Rev" : 0, "Mr" : 1, "Dr" : 2, "Master" : 3, "Miss" : 4, "Mrs" : 5}
for dataset in train_test_data:
dataset['Name'] = dataset['Name'].map(score_name)
 train.head(10 )
save_file(weights_all, 'weights_all')
 save_file(weights_for_rounding, 'weights_all')
 save_file(scores_all, 'weights_all' )<data_type_conversions>
titles = [0,1,2,3,4,5]
for title in titles:
age_to_impute = train.groupby('Name')['Age'].median() [titles.index(title)]
train.loc[(train['Age'].isnull())&(train['Name'] == title), 'Age'] = age_to_impute
train['Age'].isnull().sum()
np.max(df_sub.iloc[:,1:].to_numpy().flatten()), np.min(df_sub.iloc[:,1:].to_numpy().flatten()), df_sub.shape<prepare_output>
for title in titles:
age_to_impute = train.groupby('Name')['Age'].median() [titles.index(title)]
test.loc[(test['Age'].isnull())&(test['Name'] == title), 'Age'] = age_to_impute
test['Age'].isnull().sum()
np.mean(bestonly_raw_scores), np.min(bestonly_raw_scores), np.max(bestonly_raw_scores), len(bestonly_raw_scores )<prepare_output>
for dataset in train_test_data:
dataset['Age_bin'] = pd.cut(train['Age'], 5 )
np.mean(best_weighted_scores), np.min(best_weighted_scores), np.max(best_weighted_scores), len(best_weighted_scores )<compute_test_metric>
train[['Age_bin','Survived']].groupby(['Age_bin'], as_index=False ).mean().sort_values(by='Survived' )
np.mean(best_rounded_scores), np.min(best_rounded_scores), np.max(best_rounded_scores), len(best_rounded_scores )<define_variables>
group_names = [4,1,3,2,0]
for dataset in train_test_data:
dataset['Age'] = pd.cut(train['Age'], 5, labels=group_names )
[(c,np.round(s,3)) for c,s in zip(output_categories, scores_all)]<define_variables>
train[['Age','Survived']].groupby(['Age'], as_index=False ).mean().sort_values(by='Survived' )
[(c,np.round(s,3)) for c,s in zip(output_categories, weights_all)]<import_modules>
for dataset in train_test_data:
dataset['Embarked'] = dataset['Embarked'].fillna('S' )
import os
import re
import gc
import pickle
import random
import lightgbm as lgbm
import numpy as np
import pandas as pd<import_modules>
train[['Embarked','Survived']].groupby(['Embarked'], as_index=False ).mean().sort_values(by='Survived' )
import numpy as np
import pandas as pd
import os
import numpy as np
import matplotlib.pyplot as plt
import gensim
from nltk.corpus import brown
import random
from sklearn.model_selection import KFold
import lightgbm as lgb
import gc
from keras.callbacks.callbacks import EarlyStopping
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD
from keras.models import Sequential
from keras.layers import Dense, Activation
from keras.callbacks.callbacks import EarlyStopping
from scipy.stats import spearmanr
from nltk.corpus import wordnet as wn
import tqdm
from sklearn.model_selection import StratifiedKFold<load_pretrained>
embarked_mapping = {"S": 0, "Q": 1, "C": 2}
for dataset in train_test_data:
dataset['Embarked'] = dataset['Embarked'].map(embarked_mapping )
np.set_printoptions(suppress=True)
 tokenizer = transformers.DistilBertTokenizer.from_pretrained(".. /input/distilbertbaseuncased/")
 model = transformers.DistilBertModel.from_pretrained(".. /input/distilbertbaseuncased/" )<load_from_csv>
train['FamilySize'] = train['SibSp'] + train['Parch'] + 1
test['FamilySize'] = test['SibSp'] + test['Parch'] + 1
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
tokenizer = transformers.DistilBertTokenizer.from_pretrained(BERT_PATH+'/assets/vocab.txt')
 MAX_SEQUENCE_LENGTH = 512
train = df_train = pd.read_csv(PATH+'train.csv')
 test = df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )<set_options>
train[['FamilySize','Survived']].groupby(['FamilySize'], as_index=False ).mean().sort_values(by='Survived' )
seed(42)
 tf.random.set_seed(42)
 random.seed(42 )<define_variables>
replace_family = {8 : 0, 11 : 0, 6:1, 5: 2, 1:3, 7:4, 2 : 5, 3 :6, 4:7}
data_dir = '.. /input/google-quest-challenge/'<define_variables>
for dataset in train_test_data:
dataset['FamilySize'] = dataset['FamilySize'].apply(lambda x: replace_family.get(x))
targets = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written'
]
input_columns = ['question_title', 'question_body', 'answer']<categorify>
test['Fare'].fillna(train['Fare'].median() , inplace = True )
gc.collect()
tfidf = TfidfVectorizer(ngram_range=(1, 3))
tsvd = TruncatedSVD(n_components = 60)
 tfidf_question_title = tfidf.fit_transform([simple_prepro_tfidf(l)for l in tqdm.tqdm(train["question_title"].values)])
 tfidf_question_title_test = tfidf.transform([simple_prepro_tfidf(l)for l in tqdm.tqdm(test["question_title"].values)])
 tfidf_question_title = tsvd.fit_transform(tfidf_question_title)
 tfidf_question_title_test = tsvd.transform(tfidf_question_title_test)
 tfidf_question_body = tfidf.fit_transform([simple_prepro_tfidf(l)for l in tqdm.tqdm(train["question_body"].values)])
 tfidf_question_body_test = tfidf.transform([simple_prepro_tfidf(l)for l in tqdm.tqdm(test["question_body"].values)])
 tfidf_question_body = tsvd.fit_transform(tfidf_question_body)
 tfidf_question_body_test = tsvd.transform(tfidf_question_body_test)
 tfidf_answer = tfidf.fit_transform([simple_prepro_tfidf(l)for l in tqdm.tqdm(train["answer"].values)])
 tfidf_answer_test = tfidf.transform([simple_prepro_tfidf(l)for l in tqdm.tqdm(test["answer"].values)])
 tfidf_answer = tsvd.fit_transform(tfidf_answer)
 tfidf_answer_test = tsvd.transform(tfidf_answer_test)
 type2int = {type:i for i,type in enumerate(list(set(train["category"])))}
cate = np.identity(5)[np.array(train["category"].apply(lambda x:type2int[x])) ].astype(np.float64)
 cate_test = np.identity(5)[np.array(test["category"].apply(lambda x:type2int[x])) ].astype(np.float64 )<choose_model_class>
for dataset in train_test_data:
dataset['Fare_bin'] = pd.cut(train['Fare'], 4 )
w2v_model = gensim.models.Word2Vec(brown.sents() )<categorify>
train[['Fare_bin','Survived']].groupby(['Fare_bin'], as_index=False ).mean().sort_values(by='Survived' )
def get_word_embeddings(text):
np.random.seed(abs(hash(text)) %(10 ** 8))
words = simple_prepro(text)
 vectors = np.zeros(( len(words),100))
if len(words)==0:
vectors = np.zeros(( 1,100))
for i,word in enumerate(simple_prepro(text)) :
try:
vectors[i]=w2v_model[word]
except:
vectors[i]=np.random.uniform(-0.01, 0.01,100)
 return np.concatenate([np.max(np.array(vectors), axis=0),
np.array([min(len(text),5000)/5000,
min(text.count(" "),5000)/5000,
min(len(words),1000)/1000,
min(text.count("
"),100)/100,
min(text.count("!"),20)/20,
min(text.count("?"),20)/20])])
 question_title = [get_word_embeddings(l)for l in tqdm.tqdm(train["question_title"].values)]
question_title_test = [get_word_embeddings(l)for l in tqdm.tqdm(test["question_title"].values)]
question_body = [get_word_embeddings(l)for l in tqdm.tqdm(train["question_body"].values)]
question_body_test = [get_word_embeddings(l)for l in tqdm.tqdm(test["question_body"].values)]
answer = [get_word_embeddings(l)for l in tqdm.tqdm(train["answer"].values)]
answer_test = [get_word_embeddings(l)for l in tqdm.tqdm(test["answer"].values)]
gc.collect()
tfidf = TfidfVectorizer(ngram_range=(1, 3))
tsvd = TruncatedSVD(n_components = 60)
 tfidf_question_title = tfidf.fit_transform([simple_prepro_tfidf(l)for l in tqdm.tqdm(train["question_title"].values)])
 tfidf_question_title_test = tfidf.transform([simple_prepro_tfidf(l)for l in tqdm.tqdm(test["question_title"].values)])
 tfidf_question_title = tsvd.fit_transform(tfidf_question_title)
 tfidf_question_title_test = tsvd.transform(tfidf_question_title_test)
 tfidf_question_body = tfidf.fit_transform([simple_prepro_tfidf(l)for l in tqdm.tqdm(train["question_body"].values)])
 tfidf_question_body_test = tfidf.transform([simple_prepro_tfidf(l)for l in tqdm.tqdm(test["question_body"].values)])
 tfidf_question_body = tsvd.fit_transform(tfidf_question_body)
 tfidf_question_body_test = tsvd.transform(tfidf_question_body_test)
 tfidf_answer = tfidf.fit_transform([simple_prepro_tfidf(l)for l in tqdm.tqdm(train["answer"].values)])
 tfidf_answer_test = tfidf.transform([simple_prepro_tfidf(l)for l in tqdm.tqdm(test["answer"].values)])
 tfidf_answer = tsvd.fit_transform(tfidf_answer)
 tfidf_answer_test = tsvd.transform(tfidf_answer_test)
 type2int = {type:i for i,type in enumerate(list(set(train["category"])))}
cate = np.identity(5)[np.array(train["category"].apply(lambda x:type2int[x])) ].astype(np.float64)
 cate_test = np.identity(5)[np.array(test["category"].apply(lambda x:type2int[x])) ].astype(np.float64)
 train_features = np.concatenate([question_title, question_body, answer,
tfidf_question_title, tfidf_question_body, tfidf_answer,
cate
], axis=1)
 test_features = np.concatenate([question_title_test, question_body_test, answer_test,
tfidf_question_title_test, tfidf_question_body_test, tfidf_answer_test,
cate_test
], axis=1 )<string_transform>
train[['Fare','Survived']].groupby(['Fare'], as_index=False ).mean()
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm.tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )<compute_test_metric>
train['Cabin'].value_counts().sort_index()
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.mean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
)
 def bert_model() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 return model
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size,
fold=None)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 model.fit(train_data[0], train_data[1], epochs=epochs,
batch_size=batch_size, callbacks=[custom_callback])
 return custom_callback
gkf = GroupKFold(n_splits=10 ).split(X=df_train.question_body, groups=df_train.question_body)
 outputs = compute_output_arrays(df_train, output_categories)
 inputs = compute_input_arays(df_train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputs = compute_input_arays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH )<define_search_space>
for dataset in train_test_data:
dataset['Cabin'] = dataset['Cabin'].str[:1]
length =(np.linspace(1, 229, num=1))<import_modules>
train.loc[(train['Cabin'].isnull())&(train['Pclass']==1), 'Cabin'] = 'E'
train.loc[(train['Cabin'].isnull())&(train['Pclass']==2), 'Cabin'] = 'D'
train.loc[(train['Cabin'].isnull())&(train['Pclass']==3), 'Cabin'] = 'F'
test.loc[(test['Cabin'].isnull())&(test['Pclass']==1), 'Cabin'] = 'E'
test.loc[(test['Cabin'].isnull())&(test['Pclass']==2), 'Cabin'] = 'D'
test.loc[(test['Cabin'].isnull())&(test['Pclass']==3), 'Cabin'] = 'F'
class TextDataset(data.Dataset):
def __init__(self, text, lens, y=None):
self.text = text
self.lens = lens
self.y = y
def __len__(self):
return len(self.lens)
 def __getitem__(self, idx):
if self.y is None:
return self.text[idx], self.lens[idx]
return self.text[idx], self.lens[idx], self.y[idx]
class Collator(object):
def __init__(self, test=False, percentile=100):
self.test = test
self.percentile = percentile
def __call__(self, batch):
global MAX_LEN
if self.test:
texts, lens = zip(*batch)
 else:
texts, lens, target = zip(*batch)
 lens = np.array(lens)
 max_len = min(int(np.percentile(lens, self.percentile)) , MAX_LEN)
 texts = torch.tensor(sequence.pad_sequences(texts, maxlen=max_len), dtype=torch.long)
 if self.test:
return texts
return texts, torch.tensor(target, dtype=torch.float32)
 train_collate = Collator(percentile=100)
 train_dataset = TextDataset(train, length, test)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, collate_fn=train_collate)
 n_repeats = 10
start_time = time.time()
for _ in range(n_repeats):
for i in tqdm(lengths):
pass
method1_time =(time.time() - start_time)/ n_repeats 
<load_pretrained>
cabin_mapping = {"B": 0, "C": 1 , "A": 2, "T": 3, "E": 4, "D": 5, "F": 6, "G": 7}
for dataset in train_test_data:
dataset['Cabin'] = dataset['Cabin'].map(cabin_mapping )
train_collator = SequenceBucketCollator(lambda lengths: lengths.max() ,
sequence_index=0,
length_index=1,
label_index=2)
 test_collator = SequenceBucketCollator(lambda lengths: lengths.max() , sequence_index=0, length_index=1)
 valid_dataset = data.Subset(train, indices=[0, 1])
 train_loader = data.DataLoader(train, batch_size=batch_size, shuffle=True, collate_fn=train_collator)
 valid_loader = data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, collate_fn=train_collator)
 test_loader = data.DataLoader(test, batch_size=batch_size, shuffle=False, collate_fn=test_collator )<compute_test_metric>
features_drop = ['Ticket', 'SibSp', 'Parch', 'Age_bin', 'Fare_bin']
train = train.drop(features_drop, axis=1)
 test = test.drop(features_drop, axis=1 )
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.mean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
)
 def bert_model() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 x = tf.keras.layers.Dropout(0.2 )(x)
 y = tf.keras.layers.BatchNormalization(epsilon=1e-06, momentum=0.9, weights=None)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 return model
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size,
fold=None)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 model.fit(train_data[0], train_data[1], epochs=epochs,
batch_size=batch_size, callbacks=[custom_callback])
 return custom_callback
histories = []
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold < 3:
K.clear_session()
model = bert_model()
train_inputs = [inputs[i][train_idx] for i in range(3)]
train_outputs = outputs[train_idx]
valid_inputs = [inputs[i][valid_idx] for i in range(3)]
valid_outputs = outputs[valid_idx]
history = train_and_predict(model,
train_data=(train_inputs, train_outputs),
valid_data=(valid_inputs, valid_outputs),
test_data=test_inputs,
learning_rate=3e-5, epochs=5, batch_size=8,
loss_function='binary_crossentropy', fold=fold)
 histories.append(history )<save_to_csv>
train.to_csv('train.csv', index=False)
 test.to_csv('test.csv', index=False )
test_predictions = [histories[i].test_predictions for i in range(len(histories)) ]
test_predictions = [np.average(test_predictions[i], axis=0)for i in range(len(test_predictions)) ]
test_predictions = np.mean(test_predictions, axis=0)
 df_sub.iloc[:, 1:] = test_predictions
df_sub.to_csv('submission.csv', index=False )<load_from_csv>
train['Age'] = train['Age'].astype(int)
 test['Age'] = test['Age'].astype(int)
 train['Fare'] = train['Fare'].astype(int)
 test['Fare'] = test['Fare'].astype(int )
ROOT = '.. /input/google-quest-challenge/'
test_df = pd.read_csv(ROOT+'test.csv')
 train_df = pd.read_csv(ROOT+'train.csv' )<define_variables>
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
import numpy as np
target_cols = ['question_asker_intent_understanding', 'question_body_critical',
'question_conversational', 'question_expect_short_answer',
'question_fact_seeking', 'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written', 'answer_helpful',
'answer_level_of_information', 'answer_plausible',
'answer_relevance', 'answer_satisfaction',
'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written']<import_modules>
np.random.seed(42)
 print('tensorflow version : ', tf.__version__)
 print('keras version : ', keras.__version__ )
import torch
from torch.utils.data import TensorDataset
from torch.utils.data import DataLoader
from torch.utils.data import RandomSampler, SequentialSampler
from pytorch_transformers import BertTokenizer
from sklearn.preprocessing import MinMaxScaler<load_pretrained>
train = train.drop(['PassengerId'], axis=1 )
def read_data(raw_data_path):
test = pd.read_csv(raw_data_path, encoding='utf-8')
 targets = [-1] * len(test)
 sentence_a = test['question_title'] + test['question_body']
sentence_b = test['answer']
return targets, sentence_a, sentence_b
def save_pickle(data, file_path):

 if isinstance(file_path, Path):
file_path = str(file_path)
 with open(file_path, 'wb')as f:
pickle.dump(data, f)
 y, X_a, X_b = read_data(ROOT+'test.csv' )<concatenate>
x_data = train.values[:, 1:]
y_data = train.values[:, 0]
X_train, X_val, y_train, y_val = train_test_split(x_data, y_data, test_size = 0.3, random_state = 42 )
data = []
for step,(data_x_a, data_x_b, data_y)in enumerate(zip(X_a, X_b, y)) :
data.append(( [data_x_a, data_x_b], data_y))<load_pretrained>
model = Sequential()
model.add(Dense(255, input_shape=(8,), activation = 'relu'))
model.add(Dense(( 1), activation = 'sigmoid'))
model.compile(loss='mse', optimizer='Adam', metrics = ['accuracy'])
 model.summary()
tokenizer = BertTokenizer("/kaggle/input/bertpretrained/uncased_L-24_H-1024_A-16/uncased_L-24_H-1024_A-16/vocab.txt", True )<categorify>
hist = model.fit(X_train, y_train, validation_data=(X_val, y_val), epochs=100 )
class InputExample(object):
def __init__(self, guid, text_a, text_b=None, label=None):

 self.guid = guid
self.text_a = text_a
self.text_b = text_b
self.label = label
class InputFeature(object):

 def __init__(self, input_ids, input_mask, segment_ids, label_id, input_len):
self.input_ids = input_ids
self.input_mask = input_mask
self.segment_ids = segment_ids
self.label_id = label_id
self.input_len = input_len
def create_examples(lines, example_type):

 examples = []
for i, line in enumerate(lines):
guid = '%s-%d' %(example_type, i)
 text_a = line[0][0]
text_b = line[0][1]
label = line[1]
example = InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)
 examples.append(example)
 return examples
def truncate_seq_pair(tokens_a, tokens_b, max_length=512):
while True:
total_length = len(tokens_a)+ len(tokens_b)
 if total_length <= max_length:
break
if len(tokens_a)> len(tokens_b):
tokens_a.pop()
else:
tokens_b.pop()
def create_features(examples, max_seq_len=512):

 features = []
for ex_id, example in enumerate(examples):
tokenizer = BertTokenizer("/kaggle/input/bertpretrained/uncased_L-24_H-1024_A-16/uncased_L-24_H-1024_A-16/vocab.txt", True)
 tokens_a = tokenizer.tokenize(example.text_a)
 tokens_b = None
label_id = example.label
if example.text_b:
tokens_b = tokenizer.tokenize(example.text_b)
 truncate_seq_pair(tokens_a, tokens_b, max_length=max_seq_len - 3)
 else:
if len(tokens_a)> max_seq_len - 2:
tokens_a = tokens_a[:max_seq_len - 2]
tokens = ['[CLS]'] + tokens_a + ['[SEP]']
segment_ids = [0] * len(tokens)
 if tokens_b:
tokens += tokens_b + ['[SEP]']
segment_ids += [1] *(len(tokens_b)+ 1)
 input_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_mask = [1] * len(input_ids)
 padding = [0] *(max_seq_len - len(input_ids))
input_len = len(input_ids)
 input_ids += padding
input_mask += padding
segment_ids += padding
assert len(input_ids)== max_seq_len
assert len(input_mask)== max_seq_len
assert len(segment_ids)== max_seq_len
feature = InputFeature(input_ids=input_ids,
input_mask=input_mask,
segment_ids=segment_ids,
label_id=label_id,
input_len=input_len)
 features.append(feature)
 return features<feature_engineering>
k_fold = KFold(n_splits=10, shuffle=True, random_state=0 )
test_examples = create_examples(data, 'test')
 test_features = create_features(test_examples )<create_dataframe>
clf = KNeighborsClassifier(n_neighbors = 13)
 scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )
def create_dataset(features, is_sorted=False):
if is_sorted:
logger.info("sorted data by th length of input")
 features = sorted(features, key=lambda x: x.input_len, reverse=True)
 all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
 all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
 all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)
 all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.float)
 dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
 return dataset<load_pretrained>
round(np.mean(score)*100, 2 )
test_dataset = create_dataset(test_features)
 test_sampler = SequentialSampler(test_dataset)
 test_dataloader = DataLoader(test_dataset,sampler=test_sampler,batch_size=32 )<choose_model_class>
clf = DecisionTreeClassifier()
scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )
class BertForMultiClass(BertPreTrainedModel):
def __init__(self, config):
super(BertForMultiClass, self ).__init__(config)
 self.bert = BertModel(config)
 self.dropout = nn.Dropout(config.hidden_dropout_prob)
 self.classifier = nn.Linear(config.hidden_size, config.num_labels)
 self.apply(self.init_weights)
 def forward(self, input_ids, token_type_ids=None, attention_mask=None, head_mask=None):
outputs = self.bert(input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask,
head_mask=head_mask)
 pooled_output = outputs[1]
pooled_output = self.dropout(pooled_output)
 logits = self.classifier(pooled_output)
 return logits<categorify>
round(np.mean(score)*100, 2 )
def prepare_device(use_gpu=0):

 n_gpu_use = [int(x)for x in use_gpu.split(",")]
if not use_gpu:
device_type = 'cpu'
else:
device_type = f"cuda:{n_gpu_use[0]}"
n_gpu = torch.cuda.device_count()
if len(n_gpu_use)> 0 and n_gpu == 0:
device_type = 'cpu'
if len(n_gpu_use)> n_gpu:
msg = f"Warning: The number of GPU's configured to use is {n_gpu}, but only {n_gpu} are available on this machine."
n_gpu_use = range(n_gpu)
 device = torch.device(device_type)
 list_ids = n_gpu_use
return device, list_ids
def model_device(n_gpu, model):

 device, device_ids = prepare_device(n_gpu)
 if len(device_ids)> 1:
model = torch.nn.DataParallel(model, device_ids=device_ids)
 if len(device_ids)== 1:
os.environ['CUDA_VISIBLE_DEVICES'] = str(device_ids[0])
 model = model.to(device)
 return model, device
class Predictor(object):
def __init__(self, model, n_gpu):
self.model = model
self.model, self.device = model_device(n_gpu=n_gpu, model=self.model)
 def predict(self, data):
all_logits = None
self.model.eval()
with torch.no_grad() :
for step, batch in enumerate(data):
batch = tuple(t.to(self.device)for t in batch)
 input_ids, input_mask, segment_ids, label_ids = batch
logits = self.model(input_ids, segment_ids, input_mask)
 if all_logits is None:
all_logits = torch.sigmoid(logits ).detach().cpu().numpy()
else:
all_logits = np.concatenate([all_logits, torch.sigmoid(logits ).detach().cpu().numpy() ], axis=0)
 if 'cuda' in str(self.device):
torch.cuda.empty_cache()
return all_logits<load_pretrained>
clf = RandomForestClassifier(n_estimators=13)
 scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )
model1 = BertForMultiClass.from_pretrained("/kaggle/input/bert-fold1-new/", num_labels=30)
 model2 = BertForMultiClass.from_pretrained("/kaggle/input/bert-fold2/", num_labels=30)
 model3 = BertForMultiClass.from_pretrained("/kaggle/input/bert-fold3-new/", num_labels=30)
 model4 = BertForMultiClass.from_pretrained("/kaggle/input/bert-fold4/", num_labels=30)
 model5 = BertForMultiClass.from_pretrained("/kaggle/input/bert-fold5/", num_labels=30)
 result = []
predictor1 = Predictor(model=model1, n_gpu='0')
 result1 = predictor1.predict(data=test_dataloader)
 predictor2 = Predictor(model=model2, n_gpu='0')
 result2 = predictor2.predict(data=test_dataloader)
 predictor3 = Predictor(model=model3, n_gpu='0')
 result3 = predictor3.predict(data=test_dataloader)
 predictor4 = Predictor(model=model4, n_gpu='0')
 result4 = predictor4.predict(data=test_dataloader)
 predictor5 = Predictor(model=model5, n_gpu='0')
 result5 = predictor5.predict(data=test_dataloader)
 result.append([result1, result2, result3, result4, result5] )<feature_engineering>
round(np.mean(score)*100, 2 )
test_predictions = np.average(result[0], axis=0 )<define_variables>
clf = GaussianNB()
scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )
n=test_df['url'].apply(lambda x:('english.stackexchange.com' in x)).tolist()
spelling=[]
for x in n:
if x:
spelling.append(0.5)
 else:
spelling.append(0.)<feature_engineering>
round(np.mean(score)*100, 2 )
test_preds = test_predictions
y_train = train_df[target_cols].values
for column_ind in range(30):
curr_column = y_train[:, column_ind]
values = np.unique(curr_column)
 map_quantiles = []
for val in values:
occurrence = np.mean(curr_column == val)
 cummulative = sum(el['occurrence'] for el in map_quantiles)
 map_quantiles.append({'value': val, 'occurrence': occurrence, 'cummulative': cummulative})
 for quant in map_quantiles:
pred_col = test_preds[:, column_ind]
q1, q2 = np.quantile(pred_col, quant['cummulative']), np.quantile(pred_col, min(quant['cummulative'] + quant['occurrence'], 1))
pred_col[(pred_col >= q1)&(pred_col <= q2)] = quant['value']
test_preds[:, column_ind] = pred_col<feature_engineering>
clf = SVC(C=10)
 scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )
submission_df = pd.read_csv(ROOT+'sample_submission.csv')
 submission_df[target_cols] = test_preds
submission_df['question_type_spelling']=spelling
submission_df['answer_relevance'] = submission_df['answer_relevance'].apply(lambda x : 0.33333334326744 if x < 0.7 else x)
 submission_df<save_to_csv>
round(np.mean(score)*100,2 )
sub_file_name = 'submission.csv'
submission_df.to_csv(sub_file_name, index=False)
<install_modules>
model.fit(train_data, target)
 test_data = test.drop("PassengerId", axis=1 ).copy()
prediction = model.predict(test_data )
!pip install -q.. /input/tensorflow-determinism
!pip install -q.. /input/huggingfacetokenizers/tokenizers-0.0.11-cp36-cp36m-manylinux1_x86_64.whl
!pip uninstall --yes pytorch-transformers
!pip install -q.. /input/huggingface-transformers-master<set_options>
submission = pd.DataFrame({
"PassengerId": test["PassengerId"],
"Survived": prediction
})
 submission.to_csv('submission.csv', index=False )