Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<compute_test_metric>
pylab.rcParams['figure.figsize'] = 14,10
sns.set(color_codes=True )
def compute_spearmanr(trues, preds):
rhos = []
for col_true, col_pred in zip(trues.T, preds.T):
rhos.append(spearmanr(col_true, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.mean(rhos)
<load_from_csv>
raw_train = pd.read_csv(".. /input/train.csv")
 raw_test = pd.read_csv(".. /input/test.csv")
 df_train = raw_train.copy()
df_test = raw_test.copy()
df_total = pd.concat([df_train, df_test])
 data_cleaner = [df_train, df_test]
target_columns = list(sub_df.columns)
 results = []
for fold_i in range(1, 6):
result_df = pd.DataFrame(index=model_names, columns=target_columns[-30:] + ["ave"])
 for model_name in model_names:
for target in target_columns[-30:]:
true = train_df[target_columns].iloc[kf[fold_i-1][1]].reset_index(drop=True)
 oof = pd.read_csv(model_name + f"/oof{fold_i}.csv")
 r = spearmanr(true[target].values, oof[target].values + np.random.normal(0, 1e-7, oof.shape[0])).correlation
result_df.loc[model_name, target] = r
result_df.loc[model_name, "ave"] = np.mean(result_df.loc[model_name, target_columns[-30:]].values)
 results.append(result_df)
<feature_engineering>
for df in data_cleaner:
df.drop(["PassengerId"], axis=1, inplace=True )
def norm_with_rankdata(df):
for col_name in target_columns[-30:]:
df[col_name] = rankdata(df[col_name].values)/ len(df)
 return df
kfold = 5
model_number = len(model_names)
 true_dfs = []
for i in range(5):
oof = train_df.iloc[kf[i][1]]
true_dfs.append(oof[target_columns])
 model_df = defaultdict(lambda: [])
 for model_name in model_names:
print(model_name)
 scores = []
pred_dfs = []
for i in range(5):
pred = pd.read_csv(f"{model_name}/oof{i+1}.csv")
 pred = norm_with_rankdata(pred)
 model_df[model_name].append(pred)
 true = true_dfs[i]
rho = compute_spearmanr(true.values, pred.values)
 scores.append(rho)
 pred_dfs.append(pred.copy())
 print([f"fold{i+1}: {scores[i]:.4}" for i in range(5)])
 scores = [{} for _ in range(kfold)]
all_coeffs = []
for col_num, col_name in enumerate(target_columns[-30:]):
coefficients = np.zeros(( kfold, model_number))
print(col_name, "
")
 for fold_num,(_, vld_idx)in enumerate(kf):
print("
"+"-"*50+'
[Fold {}/{}]'.format(fold_num + 1, kfold))
def function_spearmanr(x):
true = true_dfs[fold_num].loc[:, col_name].values
pred = np.zeros(len(true))
for k,(model_name, df)in enumerate(model_df.items()):
pred += x[k] * df[fold_num].loc[:, col_name].values
return -spearmanr(true, pred ).correlation
x0 = [1 for _ in range(model_number)]
res = minimize(function_spearmanr, x0, method='nelder-mead', options={'xtol': 1e-4, 'disp': True})
 coeffs = [max(val, 0)for val in res.x]
coefficients[fold_num,:] = coeffs / np.sum(coeffs)
 print('
final vector: {}'.format(coefficients[fold_num,:]))
y_true = true_dfs[fold_num].loc[:, col_name].values
y_pred = np.zeros(len(y_true))
for k,(model_name, df)in enumerate(model_df.items()):
y_pred += coefficients[fold_num, k] * df[fold_num].loc[:, col_name].values
score = spearmanr(y_true, y_pred ).correlation
scores[fold_num][col_name] = score
print(f"valid's spearmanr: {score:<7.4f}
")
 all_coeffs.append(coefficients)
<load_from_csv>
for df in data_cleaner:
df["Age"].fillna(df_train.Age.median() , inplace=True)
 df["Fare"].fillna(df_train.Fare.median() , inplace=True)
 df["Embarked"].fillna(df_train.Embarked.mode() [0], inplace=True)
 df["Cabin"].fillna('M', inplace=True )
sub_df = pd.read_csv(".. /input/google-quest-challenge/sample_submission.csv")
 submission = np.zeros(( len(sub_df), 30)).T
FOLD_NUM = 5
inf_sub = []
for fold_num in range(FOLD_NUM):
l = []
for model_num, model_name in enumerate(model_names):
sub_ = pd.read_csv(f"{model_name}/submission{fold_num+1}.csv")
 sub_ = norm_with_rankdata(sub_)
 l.append(sub_.copy())
 inf_sub.append(l.copy() )<feature_engineering>
print('-'*20, 'Train Set')
 print(df_train.isnull().any())
 print('-'*20, 'Test Set')
 print(df_test.isnull().any() )
for col_num, col_name in enumerate(target_columns[-30:]):
for fold_num in range(FOLD_NUM):
for model_num in range(len(model_names)) :
coef = all_coeffs[col_num][fold_num][model_num]
submission[col_num] += coef / FOLD_NUM * inf_sub[fold_num][model_num][col_name].values
sub_df.iloc[:, -30:] = submission.T
sub_df<feature_engineering>
for df in data_cleaner:
df["FamilyName"] = df.Name.apply(extract_name)
 df["Title"] = df.Name.apply(extract_title )
def norm_sub(df):
for col_name in df.columns[-30:]:
tmp_df = df[col_name].values
v_max = np.max(tmp_df)+ 0.01
v_min = np.min(tmp_df)- 0.01
df[col_name] = df[col_name].apply(lambda x:(x - v_min)/(v_max - v_min))
df[col_name] = df[col_name].values + np.random.normal(0, 1e-7, len(df))
return df
sub_df = norm_sub(sub_df )<load_from_csv>
df_train.FamilyName.value_counts().head(10 )
<compute_train_metric>
df_train['FamilyName'].value_counts()
def compute_actual_spearmanr(trues, preds):
rhos = []
for col_true, col_pred in zip(trues.T, preds.T):
rhos.append(spearmanr(col_true, col_pred ).correlation)
 return np.mean(rhos)
 class OptimizedRounder(object):
def __init__(self, n):
self.coef_ = 0
self.n = n
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
continue
for i in range(self.n-2):
if coef[i] <= pred and pred < coef[i+1]:
X_p[i] =(i + 1)/(self.n - 1)
 break
else:
X_p[i] = 1
ll = spearmanr(y, X_p ).correlation
return -ll
def fit(self, X, y, init_coef=None):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 if init_coef == None:
initial_coef = [1/self.n *(x+1)for x in range(self.n-1)]
else:
initial_coef = init_coef
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
continue
for i in range(self.n-2):
if coef[i] <= pred and pred < coef[i+1]:
X_p[i] =(i + 1)/(self.n - 1)
 else:
X_p[i] = 1
return X_p
def coefficients(self):
return self.coef_['x']<statistical_test>
for df in data_cleaner:
df.drop(["FamilyName"], axis=1, inplace=True )
class color:
PURPLE = '\033[95m'
CYAN = '\033[96m'
DARKCYAN = '\033[36m'
BLUE = '\033[94m'
GREEN = '\033[92m'
YELLOW = '\033[93m'
RED = '\033[91m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
END = '\033[0m'
def compute_spearmanr(trues, preds, columns=None):
rhos = []
for i,(col_true, col_pred)in enumerate(zip(trues.T, preds.T)) :
if i == 0:
continue
r = spearmanr(col_true, col_pred ).correlation
if columns:
print("{:.3f}\t{}".format(r, columns[i-1]))
rhos.append(r)
 return np.mean(rhos)
 def compare_spearmanr(trues, preds, pp_preds, columns=None):
rhos = []
pp_rhos = []
for i,(col_true, col_pred, col_pp_pred)in enumerate(zip(trues.T, preds.T, pp_preds.T)) :
if i == 0:
continue
r = spearmanr(col_true, col_pred ).correlation
pp_r = spearmanr(col_true, col_pp_pred ).correlation
if columns:
if r < pp_r:
print("{:.3f} ->".format(r)+ color.BOLD + " {:.3f}".format(pp_r)+ color.END + " \t{}".format(columns[i-1]))
elif(r - pp_r)< 10**(-6):
print("{:.3f} ->".format(r)+ " {:.3f}".format(pp_r)+ " \t{}".format(columns[i-1]))
else:
print("{:.3f} ->".format(r)+ color.RED + " {:.3f}".format(pp_r)+ color.END + " \t{}".format(columns[i-1]))
rhos.append(r)
 pp_rhos.append(pp_r)
 whole_base = np.mean(rhos)
 whole_pp = np.mean(pp_rhos)
 print("Whole Score")
 if whole_base < whole_pp:
print("{:.3f} ->".format(whole_base)+ color.BOLD + " {:.3f}".format(whole_pp)+ color.END + " +{}".format(abs(whole_pp - whole_base)))
 elif(whole_base - whole_pp)< 10**(-6):
print("{:.3f} ->".format(whole_base)+ " {:.3f}".format(whole_pp))
else:
print("{:.3f} ->".format(whole_base)+ color.RED + " {:.3f}".format(whole_pp)+ color.END + " -{}".format(abs(whole_pp - whole_base)))
 return whole_base, whole_pp
def inf_numbert_of_annotators(v):
s = set(v)
 d = 10
for v in s:
for u in s:
if v != u:
d = min(abs(v - u), d)
 for i in range(100):
if d * i >= 1 - 10**(-3):
return i
def zero_one_scale(v):
max_x = np.max(v)
 min_x = np.min(v)
 return(v - min_x)/(max_x - min_x)
 oofs = []
for fold_num in range(FOLD_NUM):
oof_ = np.zeros(( len(model_df[model_names[0]][fold_num]), 30)).T
for col_num, col_name in enumerate(target_columns[-30:]):
for model_num, model_name in enumerate(model_names):
coef = all_coeffs[col_num][fold_num][model_num]
oof_[col_num] += coef * model_df[model_name][fold_num][col_name].values
oofs.append(oof_.T)
 gold_label_df = train_df[target_columns]
pred_label_df = gold_label_df.copy()
for fold_num in range(5):
pred_label_df.loc[kf[fold_num][1], target_columns[1:]] = oofs[fold_num]
pred_label_df = norm_with_rankdata(pred_label_df)
 def post_process(train_df, train, pred_df):
df = pred_df.copy()
df.loc[(train["host"] != "english.stackexchange.com")&(train["host"] != "ell.stackexchange.com"), "question_type_spelling"] = 0
df = norm_with_rankdata(df)
 for col_name in target_columns[1:]:
d = defaultdict(int)
 for v in gold_label_df[col_name].values:
d[v] += 1
d = sorted(d.items())
 data_length = len(gold_label_df)
 coefs = []
for i,(_, v)in enumerate(d[:-1]):
v_ = v / data_length
if i > 0:
v_ += coefs[i-1]
coefs.append(v_)
 n = inf_numbert_of_annotators(train_df[col_name].values)
 gold = gold_label_df[col_name].values
pred = pred_label_df[col_name].values
base_score = spearmanr(gold, pred ).correlation
optR = OptimizedRounder(len(d))
optR.fit(pred, gold, coefs)
 opt_v = optR.predict(pred, optR.coefficients())
 opt_score = spearmanr(gold, opt_v ).correlation
if base_score < opt_score and n < 10:
df[col_name] = optR.predict(df[col_name].values, coefs)
 if len(set(df[col_name].values)) == 1:
df[col_name] = pred_df[col_name].values
df.loc[(( train["host"] == "english.stackexchange.com")|(train["host"] == "ell.stackexchange.com")) , "question_type_spelling"] += 0.1
df["question_type_spelling"] = zero_one_scale(df["question_type_spelling"].values)
 return df<load_from_csv>
titles = df_train.Title
train_df = pd.read_csv('.. /input/google-quest-challenge/train.csv')
 test_df = pd.read_csv('.. /input/google-quest-challenge/test.csv')
 sub_df = post_process(train_df, test_df, sub_df )<compute_test_metric>
titles.value_counts()
cv_score = 0
for fold_num in range(FOLD_NUM):
fold_base = model_df[model_names[0]][fold_num].copy()
fold_base.iloc[:, -30:] = oofs[fold_num]
r = compute_spearmanr(true_dfs[fold_num].iloc[:, -30:].values, fold_base.iloc[:, -30:].values)
 cv_score += r / FOLD_NUM
print(cv_score )<compute_test_metric>
miss = ["Ms", "Mlle"]
mrs = ["Mme"]
for df in data_cleaner:
df["Title"] = df.Title.apply(lambda f: 'Miss' if f in miss else f)
 df["Title"] = df.Title.apply(lambda f: 'Mrs' if f in mrs else f )
cv_score = 0
for fold_num in range(FOLD_NUM):
fold_base = model_df[model_names[0]][fold_num].copy()
fold_base.iloc[:, -30:] = oofs[fold_num]
fold_base = post_process(train_df, train_df.iloc[kf[fold_num][1]].reset_index(drop=True), fold_base)
 r = compute_actual_spearmanr(true_dfs[fold_num].iloc[:, -30:].values, fold_base.iloc[:, -30:].values)
 cv_score += r / FOLD_NUM
print(cv_score )<filter>
titles = df_train.Title
for df in data_cleaner:
df["Title"] = df.Title.apply(lambda f: f if f in titles.unique() and
titles.value_counts() [f] > 10 else 'Rare' )
sub_df[sub_df["question_type_spelling"] > 0]<save_to_csv>
df_train['Title'].value_counts()
sub_df.to_csv("submission.csv", index=False )<import_modules>
_, fare_bins = pd.qcut(df_total['Fare'], 4, retbins=True)
 fare_bins[0] -= 0.001
_, age_bins = pd.cut(df_total['Age'], 5, retbins=True)
 for df in data_cleaner:
df["FareBin"] = pd.cut(df['Fare'], fare_bins)
 df["AgeBin"] = pd.cut(df['Age'], age_bins )
pyLDAvis.enable_notebook()
np.random.seed(2018)
 warnings.filterwarnings('ignore')
 np.set_printoptions(suppress=True )<define_variables>
for df in data_cleaner:
df['IsChild'] = df['Age'] < 16
input_columns = ['question_title', 'question_body', 'answer']
targets = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written'
]<load_from_csv>
for df in data_cleaner:
df["FamilySize"] = df.Parch + df.SibSp +1
train = pd.read_csv('/kaggle/input/google-quest-challenge/train.csv',index_col='qa_id')
 test = pd.read_csv('/kaggle/input/google-quest-challenge/test.csv',index_col='qa_id')
 submission = pd.read_csv('/kaggle/input/google-quest-challenge/sample_submission.csv')
 train.shape,test.shape,submission.shape<groupby>
for df in data_cleaner:
df["IsAlone"] = df["FamilySize"] == 1
df["LargeFamily"] = df["FamilySize"] >= 5
<load_pretrained>
ticket_values = pd.concat([df_train['Ticket'], df_test['Ticket']] ).value_counts()
ticket_values.head()
<load_pretrained>
for df in data_cleaner:
df['N_ticket'] = df['Ticket'].apply(lambda f: ticket_values[f] )
%%time
def fetch_vectors(string_list, batch_size=64):
DEVICE = torch.device("cuda")
 tokenizer = transformers.BertTokenizer.from_pretrained(".. /input/bertbaseuncased/bert-base-uncased/")
 model = transformers.BertModel.from_pretrained(".. /input/bertbaseuncased/bert-base-uncased/")
 model.to(DEVICE)
 fin_features = []
for data in chunks(string_list, batch_size):
tokenized = []
for x in data:
x = " ".join(x.strip().split() [:300])
 tok = tokenizer.encode(x, add_special_tokens=True)
 tokenized.append(tok[:512])
 max_len = 512
padded = np.array([i + [0] *(max_len - len(i)) for i in tokenized])
 attention_mask = np.where(padded != 0, 1, 0)
 input_ids = torch.tensor(padded ).to(DEVICE)
 attention_mask = torch.tensor(attention_mask ).to(DEVICE)
 with torch.no_grad() :
last_hidden_states = model(input_ids, attention_mask=attention_mask)
 features = last_hidden_states[0][:, 0, :].cpu().numpy()
fin_features.append(features)
 fin_features = np.vstack(fin_features)
 return fin_features
def chunks(l, n):

 for i in range(0, len(l), n):
yield l[i:i + n]
sys.path.insert(0, ".. /input/transformers/transformers-master/")
 train_question_body_dense = fetch_vectors(train.question_body.values)
 train_answer_dense = fetch_vectors(train.answer.values)
 test_question_body_dense = fetch_vectors(test.question_body.values)
 test_answer_dense = fetch_vectors(test.answer.values )<categorify>
for df in data_cleaner:
df["Cabin"] = df["Cabin"].apply(lambda f: f[0] )
%%time
find = re.compile(r"^[^.]*")
 train['netloc'] = train['url'].apply(lambda x: re.findall(find, urlparse(x ).netloc)[0])
 test['netloc'] = test['url'].apply(lambda x: re.findall(find, urlparse(x ).netloc)[0])
 features = ['netloc', 'category']
merged = pd.concat([train[features], test[features]])
 ohe = OneHotEncoder()
ohe.fit(merged)
 features_train = ohe.transform(train[features] ).toarray()
features_test = ohe.transform(test[features] ).toarray()<load_from_csv>
for df in data_cleaner:
df['CabinMissing'] = df.Cabin == 'M'
<load_pretrained>
for df in data_cleaner:
print('-'*20)
 print(df.isnull().any() )
%%time
sys.path.insert(0, "/kaggle/input/tftext/tensorflow_text/")
 embed = hub.load("/kaggle/input/useqa3/USEQA3/" )<feature_engineering>
label = LabelEncoder()
for df in data_cleaner:
df['Sex_Code'] = label.fit_transform(df['Sex'])
 df['Embarked_Code'] = label.fit_transform(df['Embarked'])
 df['Title_Code'] = label.fit_transform(df['Title'])
 df['AgeBin_Code'] = label.fit_transform(df['AgeBin'])
 df['FareBin_Code'] = label.fit_transform(df['FareBin'])
 df['Cabin_Code'] = label.fit_transform(df["Cabin"])
 df['Pclass'] = df.Pclass.astype('category')
 df['IsChild'] = df['IsChild'].apply(int)
 df['IsAlone'] = df['IsAlone'].apply(int)
 df['LargeFamily'] = df['LargeFamily'].apply(int)
 df['CabinMissing'] = df['CabinMissing'].apply(int )
%%time
embeddings_train = {}
embeddings_test = {}
print("preparing embeddings for train data.... ")
 train['question_title']=train['question_title'].apply(lambda x:x.strip('
'))
train['question_body']=train['question_body'].apply(lambda x:x.strip('
'))
train['answer']=train['answer'].apply(lambda x:x.strip('
'))
train_ans_emb = []
train_questitle_emb = []
train_quesbody_emb = []
for q_t,q_b,a in tqdm(zip(list(train['question_title']),list(train['question_body']),list(train['answer']))):
question_title=[q_t]
question_body=[q_b]
responses=[a]
response_contexts = responses
question_title_embeddings = embed.signatures['question_encoder'](tf.constant(question_title)) ['outputs']
question_body_embeddings = embed.signatures['question_encoder'](tf.constant(question_body)) ['outputs']
response_embeddings = embed.signatures['response_encoder'](input=tf.constant(responses),context=tf.constant(response_contexts)) ['outputs']
train_ans_emb.append(response_embeddings.numpy())
 train_questitle_emb.append(question_title_embeddings.numpy())
 train_quesbody_emb.append(question_body_embeddings.numpy())
 embeddings_train['answer_embedding'] = np.vstack(train_ans_emb)
 embeddings_train['question_body_embedding'] = np.vstack(train_quesbody_emb)
 embeddings_train['question_title_embedding'] = np.vstack(train_questitle_emb)
 print("preparing embeddings for test data.... ")
 test['question_title']=test['question_title'].apply(lambda x:x.strip('
'))
test['question_body']=test['question_body'].apply(lambda x:x.strip('
'))
test['answer']=test['answer'].apply(lambda x:x.strip('
'))
test_ans_emb = []
test_questitle_emb = []
test_quesbody_emb = []
for q_t,q_b,a in tqdm(zip(list(test['question_title']),list(test['question_body']),list(test['answer']))):
question_title=[q_t]
question_body=[q_b]
responses=[a]
response_contexts = responses
question_title_embeddings = embed.signatures['question_encoder'](tf.constant(question_title)) ['outputs']
question_body_embeddings = embed.signatures['question_encoder'](tf.constant(question_body)) ['outputs']
response_embeddings = embed.signatures['response_encoder'](input=tf.constant(responses),context=tf.constant(response_contexts)) ['outputs']
test_ans_emb.append(response_embeddings.numpy())
 test_questitle_emb.append(question_title_embeddings.numpy())
 test_quesbody_emb.append(question_body_embeddings.numpy())
 embeddings_test['answer_embedding'] = np.vstack(test_ans_emb)
 embeddings_test['question_body_embedding'] = np.vstack(test_quesbody_emb)
 embeddings_test['question_title_embedding'] = np.vstack(test_questitle_emb)
 del embed
K.clear_session()
gc.collect()<concatenate>
target = ['Survived']
data_pretty = ['Age', 'Pclass', 'Title', 'Sex', 'SibSp', 'Parch', 'Fare', 'Cabin', 'Embarked',
'FamilySize', 'FareBin', 'AgeBin', 'IsAlone', 'IsChild', 'LargeFamily',
'N_ticket', 'CabinMissing']
data_numbers = ['Age', 'SibSp', 'Parch', 'Fare', 'FamilySize', 'N_ticket']
data_bins = ['AgeBin_Code', 'FareBin_Code']
data_cat = ['Pclass', 'Title', 'Sex', 'Cabin', 'Embarked' 'IsAlone',
'CabinMissing', 'IsChild', 'LargeFamily']
data_code = ['Pclass', 'Title_Code', 'Sex_Code', 'Cabin_Code', 'Embarked_Code',
'IsAlone', 'CabinMissing', 'IsChild', 'LargeFamily']
l2_dist = lambda x, y: np.power(x - y, 2 ).sum(axis=1)
 cos_dist = lambda x, y:(x*y ).sum(axis=1)
 dist_features_train = np.array([
l2_dist(embeddings_train['question_title_embedding'], embeddings_train['answer_embedding']),
l2_dist(embeddings_train['question_body_embedding'], embeddings_train['answer_embedding']),
l2_dist(embeddings_train['question_body_embedding'], embeddings_train['question_title_embedding']),
cos_dist(embeddings_train['question_title_embedding'], embeddings_train['answer_embedding']),
cos_dist(embeddings_train['question_body_embedding'], embeddings_train['answer_embedding']),
cos_dist(embeddings_train['question_body_embedding'], embeddings_train['question_title_embedding'])
 ] ).T
dist_features_test = np.array([
l2_dist(embeddings_test['question_title_embedding'], embeddings_test['answer_embedding']),
l2_dist(embeddings_test['question_body_embedding'], embeddings_test['answer_embedding']),
l2_dist(embeddings_test['question_body_embedding'], embeddings_test['question_title_embedding']),
cos_dist(embeddings_test['question_title_embedding'], embeddings_test['answer_embedding']),
cos_dist(embeddings_test['question_body_embedding'], embeddings_test['answer_embedding']),
cos_dist(embeddings_test['question_body_embedding'], embeddings_test['question_title_embedding'])
 ] ).T
X_train = np.hstack([item for k, item in embeddings_train.items() ] + [features_train,dist_features_train])
 X_test = np.hstack([item for k, item in embeddings_test.items() ] + [features_test,dist_features_test])
 y_train = train[targets].values<concatenate>
dummy_train = pd.get_dummies(df_train[data_pretty + target])
 dummy_test = pd.get_dummies(df_test[data_pretty])
 dummy_labels = dummy_test.columns.tolist()
dummy = [dummy_train, dummy_test]
X_train = np.hstack(( X_train, train_question_body_dense, train_answer_dense))
X_test = np.hstack(( X_test, test_question_body_dense, test_answer_dense))
X_train.shape,X_test.shape,y_train.shape 
<save_to_csv>
for x in data_pretty:
if df_train[x].dtype != 'float64' :
print('Survival Correlation by:', x)
 print(df_train[[x, target[0]]].groupby(x, as_index=False ).mean())
 print('-'*10, '
' )
pd.DataFrame(X_train ).to_csv('X_train_USEQA_BERTuncased.csv')
 pd.DataFrame(y_train ).to_csv('y_train_USEQA_BERTuncased.csv')
 pd.DataFrame(X_test ).to_csv('X_test_USEQA_BERTuncased.csv' )<prepare_x_and_y>
dummy_test['Cabin_T'] = 0
class SpearmanRhoCallback(Callback):
def __init__(self, training_data, validation_data, patience, model_name):
self.x = training_data[0]
self.y = training_data[1]
self.x_val = validation_data[0]
self.y_val = validation_data[1]
self.patience = patience
self.value = -1
self.bad_epochs = 0
self.model_name = model_name
def on_train_begin(self, logs={}):
return
def on_train_end(self, logs={}):
return
def on_epoch_begin(self, epoch, logs={}):
return
def on_epoch_end(self, epoch, logs={}):
y_pred_val = self.model.predict(self.x_val)
 rho_val = np.mean([spearmanr(self.y_val[:, ind], y_pred_val[:, ind] + np.random.normal(0, 1e-7, y_pred_val.shape[0])).correlation for ind in range(y_pred_val.shape[1])])
 if rho_val >= self.value:
self.value = rho_val
else:
self.bad_epochs += 1
if self.bad_epochs >= self.patience:
print("Epoch %05d: early stopping Threshold" % epoch)
 self.model.stop_training = True
print('\rval_spearman-rho: %s' %(str(round(rho_val, 4))), end=100*' '+'
')
 return rho_val
def on_batch_begin(self, batch, logs={}):
return
def on_batch_end(self, batch, logs={}):
return<choose_model_class>
features = ['IsChild',
'IsAlone',
'LargeFamily',
'SibSp', 'Parch',
'FamilySize',
'N_ticket',
'Title_Rare',
'Sex_female',
'Age',
'Fare',
"Pclass_1", "Pclass_2",
"Embarked_C", "Embarked_S",
'AgeBin_(16.136, 32.102]', 'AgeBin_(32.102, 48.068]',
'AgeBin_(48.068, 64.034]', 'AgeBin_(64.034, 80.0]',
"FareBin_(7.896, 14.454]", "FareBin_(14.454, 31.275]", "FareBin_(31.275, 512.329]",
'CabinMissing']
features1 = ['IsChild', 'IsAlone', "LargeFamily", 'SibSp', 'Parch', 'Sex_female',"Pclass_1", "Pclass_2",
"Embarked_C", "Embarked_S", 'CabinMissing', 'Age', 'Fare']
features2 = ['IsChild', 'IsAlone', 'SibSp', 'Parch', 'Sex_female',"Pclass_1", "Pclass_2",
"Embarked_C", "Embarked_S", 'CabinMissing', 'Age', "LargeFamily",
"FareBin_(7.896, 14.454]", "FareBin_(14.454, 31.275]", "FareBin_(31.275, 512.329]"]
features3 = ['IsChild', 'IsAlone', 'SibSp', 'Parch', 'Sex_female',"Pclass_1", "Pclass_2",
"Embarked_C", "Embarked_S", 'CabinMissing', "LargeFamily"]
features4 = ['Sex_female',"Pclass_1", "Pclass_2", 'IsChild', 'FamilySize', 'Fare',
"Embarked_C", "Embarked_S", 'CabinMissing', "LargeFamily"]
feature_options = [features1, features2, features3, features4]
def create_model(n_dense1=256,dropout1=0.30,lr_rate=0.00003):
model = Sequential()
model.add(Dense(n_dense1, input_dim=X_train.shape[1], activation='elu'))
model.add(Dropout(dropout1))
model.add(Dense(y_train.shape[1], activation='sigmoid'))
model.compile(optimizer=tf.keras.optimizers.Adam(lr=lr_rate),loss=tf.keras.losses.binary_crossentropy,metrics=['accuracy'])
 model.summary()
return model<train_model>
train_set = dummy_train[features + target].copy()
test_set = dummy_test[features].copy()
for e in train_set.columns:
if e in data_numbers:
test_set[e] = StandardScaler().fit_transform(test_set[e].values.reshape(-1,1)).ravel()
%%time
all_predictions = []
kf = KFold(n_splits=5, random_state=42, shuffle=True)
 for ind,(tr, val)in enumerate(kf.split(X_train)) :
X_tr = X_train[tr]
y_tr = y_train[tr]
X_vl = X_train[val]
y_vl = y_train[val]
model = create_model()
print(X_tr.shape,y_tr.shape,X_vl.shape,y_vl.shape)
 model.fit(
 X_tr, y_tr, epochs=100, batch_size=32, validation_data=(X_vl, y_vl), verbose=False,
callbacks=[SpearmanRhoCallback(training_data=(X_tr, y_tr), validation_data=(X_vl, y_vl),
patience=5, model_name=f'best_model_batch{ind}.h5')] 
)
 all_predictions.append(model.predict(X_test)) 
<train_model>
stats.chisqprob = lambda chisq, df: stats.chi2.sf(chisq, df)
 for f in feature_options:
print("
", "Features: ", f)
 logit_model=sm.Logit(train_set[target], train_set[f])
 result=logit_model.fit()
print(result.summary())
 print("-"*20)
model = create_model()
model.fit(X_train, y_train, epochs=33, batch_size=32, verbose=False)
 all_predictions.append(model.predict(X_test))<train_on_grid>
features_ = features2
logit_model=sm.Logit(train_set[target], train_set[features_])
 result=logit_model.fit()
print(result.summary())
 print("-"*20 )
%%time
kf = KFold(n_splits=5, random_state=2019, shuffle=True)
 for ind,(tr, val)in enumerate(kf.split(X_train)) :
X_tr = X_train[tr]
y_tr = y_train[tr]
X_vl = X_train[val]
y_vl = y_train[val]
model = MultiTaskElasticNet(alpha=0.001, random_state=42, l1_ratio=0.5)
 model.fit(X_tr, y_tr)
 all_predictions.append(model.predict(X_test))<train_model>
logreg = LogisticRegression()
rfe = RFECV(logreg, 1, 10, verbose=3)
 X_rfe = rfe.fit_transform(train_set[features], train_set[target] )
%%time
model = MultiTaskElasticNet(alpha=0.001, random_state=42, l1_ratio=0.5)
 model.fit(X_train, y_train)
 all_predictions.append(model.predict(X_test))
len(all_predictions )<prepare_output>
logit_model=sm.Logit(train_set[target], train_set[features_rfe])
 result=logit_model.fit()
print(result.summary())
 print("-"*20 )
%%time
test_preds = np.array([np.array([rankdata(c)for c in p.T] ).T for p in all_predictions] ).mean(axis=0)
 max_val = test_preds.max() + 1
test_preds = test_preds/max_val + 1e-12<string_transform>
logit_model=sm.Logit(train_set[target], train_set[features2])
 result=logit_model.fit()
print(result.summary())
 print("-"*20 )
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length, t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32), np.asarray(input_masks, dtype=np.int32), np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )<predict_on_test>
X, X_test, y, y_test = train_test_split(train_set[features_].values, train_set[target].values,
test_size=0.25, stratify=train_set[target].values, random_state=42)
 X.shape, X_test.shape, y.shape, y_test.shape
class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(self.model.predict(self.test_inputs, batch_size=self.batch_size)) 
<compute_test_metric>
grid.fit(X, y.ravel() )
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.mean(rhos)
<choose_model_class>
grid.best_score_
def train_bert_model(input_units,output_units):
input_word_ids = tf.keras.layers.Input(( input_units,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(( input_units,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(( input_units,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(output_units, activation="sigmoid",name="dense_output" )(x)
 model = tf.keras.models.Model(inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 return model
def train_and_predict(model, train_data, valid_data, test_data, learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(( valid_data[0], valid_data[1]),test_data,batch_size,fold=None)
 optimizer = tf.keras.optimizers.Adam(learning_rate)
 model.compile(loss_function,optimizer)
 model.fit(train_data[0], train_data[1],epochs,batch_size, callbacks=[custom_callback])
 return custom_callback<define_variables>
grid.best_params_
<define_variables>
grid.score(X_test, y_test )
test_predictions = []
test_predictions.append(test_preds)
 final_predictions = np.mean(test_predictions, axis=0)
 final_predictions.shape<save_to_csv>
lg = grid.best_estimator_
submission.iloc[:,1:] = final_predictions
submission.to_csv('submission.csv', index=False)
 submission.head()<load_from_csv>
lg.fit(train_set[features_], train_set[target] )
<define_variables>
pred = lg.predict(test_set[features_] )
krig.seed_everything()<set_options>
df_pred = pd.concat([raw_test["PassengerId"], pd.Series(pred, name="Survived")], axis=1 )
pd.set_option('use_inf_as_na', True)
 pd.set_option('display.max_columns', 999)
 pd.set_option('display.max_rows', 999 )<define_variables>
df_pred['Survived'] = df_pred.Survived.astype(int)
 df_pred.head()
IS_KAGGLE = True
QUESTION_MODEL_NAME = 'question_bert_base_uncased_20200210_191831'
ANSWER_MODEL_NAME = 'answer_bert_base_uncased_20200210_202316'
MAX_SEQUENCE_LENGTH = 512
STRIDE = 50
WINDOW_LENGTH = 100
QUESTION_LABELS = [
'question_asker_intent_understanding', 'question_body_critical', 'question_conversational',
'question_expect_short_answer', 'question_fact_seeking', 'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self', 'question_multi_intent',
'question_not_really_a_question', 'question_opinion_seeking', 'question_type_choice', 'question_type_compare',
'question_type_consequence', 'question_type_definition', 'question_type_entity', 'question_type_instructions',
'question_type_procedure', 'question_type_reason_explanation', 'question_type_spelling', 'question_well_written',
]
ANSWER_LABELS = [
'answer_helpful', 'answer_level_of_information', 'answer_plausible', 'answer_relevance', 'answer_satisfaction',
'answer_type_instructions', 'answer_type_procedure', 'answer_type_reason_explanation', 'answer_well_written'
]
QUESTION_FIRST_SEQUENCE = ['category', 'question_title']
ANSWER_FIRST_SEQUENCE = ['category', 'question_title']
QUESTION_SECOND_SEQUENCE = ['question_body']
ANSWER_SECOND_SEQUENCE = ['answer']
BASE_DIR = '.. /resources'
if IS_KAGGLE:
BASE_DIR = '.. /input/googlequestchallenge/kaggle-google-quest-challenge-1.0/resources'<load_from_csv>
df_pred.to_csv("out.csv", index=False )