Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<train_model>
%matplotlib inline
def train_and_evaluate(train, valid, feature_engineering = False):
clfs = list()
num=1
TARGET = 'answered_correctly'
FEATURES = [
'prior_question_elapsed_time',
'prior_question_had_explanation',
'part',
'answered_correctly_u_avg',
'elapsed_time_u_avg',
'explanation_u_avg',
'answered_correctly_q_avg',
'elapsed_time_q_avg',
'explanation_q_avg',
'answered_correctly_uq_count',
'timestamp_u_recency_1',
'timestamp_u_recency_2',
'timestamp_u_recency_3',
'timestamp_u_incorrect_recency'
]
gc.collect()
print(f'Traning with {train.shape[0]} rows and {len(FEATURES)} features')
 drop_cols = list(set(train.columns)- set(FEATURES))
y_train = train[TARGET]
y_val = valid[TARGET]
train.drop(drop_cols, axis = 1, inplace = True)
 valid.drop(drop_cols, axis = 1, inplace = True)
 gc.collect()
for i in range(0,num):
lgb_train = lgb.Dataset(train[FEATURES], y_train)
 lgb_valid = lgb.Dataset(valid[FEATURES], y_val)
 del train, y_train
gc.collect()
params= {'objective': 'binary',
'metric': 'auc',
'verbosity': -1,
'boosting_type': 'gbdt',
'feature_pre_filter': False,
'learning_rate': 0.05,
'lambda_l1': 0.0003344679079114925,
'lambda_l2': 1.9095994876812776,
'num_leaves': 433,
'max_bin': 511,
'feature_fraction': 1.0,
'bagging_fraction': 1.0,
'bagging_freq': 0,
'min_child_samples': 20
}
paramsf = {'objective': 'binary',
'seed': SEED,
'metric': 'auc',
'num_leaves': 433,
'learning_rate': 0.05,
'feature_fraction': 0.75,
'bagging_freq': 10,
'bagging_fraction': 0.80
}
model = lgb.train(
 params = params,
train_set = lgb_train,
num_boost_round = 10000,
valid_sets = [lgb_train, lgb_valid],
early_stopping_rounds = 10,
verbose_eval = 50 
)
 clfs.append(model)
 print('Our Roc Auc score for the validation data is:', roc_auc_score(y_val, model.predict(valid[FEATURES])))
 feature_importance = model.feature_importance()
feature_importance = pd.DataFrame({'Features': FEATURES, 'Importance': feature_importance} ).sort_values('Importance', ascending = False)
 fig = plt.figure(figsize =(10, 10))
fig.suptitle('Feature Importance', fontsize = 20)
 plt.tick_params(axis = 'x', labelsize = 12)
 plt.tick_params(axis = 'y', labelsize = 12)
 plt.xlabel('Importance', fontsize = 15)
 plt.ylabel('Features', fontsize = 15)
 sns.barplot(x = feature_importance['Importance'], y = feature_importance['Features'], orient = 'h')
 plt.show()
return TARGET, FEATURES, model,clfs<compute_test_metric>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 train_df.head()
TARGET, FEATURES, model,clfs = train_and_evaluate(train, valid, feature_engineering = True )<choose_model_class>
train_df.isnull().sum() , print('------'),test_df.isnull().sum()
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):
super(SAKTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.embedding = nn.Embedding(2*n_skill+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = x + pos_x
e = self.e_embedding(question_ids)
 x = x.permute(1, 0, 2)
 e = e.permute(1, 0, 2)
 att_mask = future_mask(x.size(0)).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 x = self.pred(x)
 return x.squeeze(-1), att_weight<load_pretrained>
sex_map = {'male' : 0, 'female' : 1}
train_df['Sex'] = train_df['Sex'].replace(sex_map)
 fare_map = {'Unknown' : 0,'1-20' : 1,'21-41' : 2,'42-60' :3 ,'61-81' : 4,'82-100' : 5,'101+' : 6}
train_df['FareGroup'] = train_df['FareGroup'].replace(fare_map )
skills = joblib.load("/kaggle/input/riiid-sakt-model-dataset-public/skills.pkl.zip")
 n_skill = len(skills)
 group = joblib.load("/kaggle/input/riiid-sakt-model-dataset-public/group.pkl.zip" )<load_pretrained>
def age_imputer(dataf_to_impute,dataf_to_ref):
title_age = dataf_to_ref[['Title','Age']][dataf_to_ref['Age'].notnull() ].groupby('Title' ).mean()
for Id in dataf_to_impute['PassengerId'][dataf_to_impute['Age'].isnull() ]:
for tle in dataf_to_impute['Title'][dataf_to_impute['PassengerId'] == Id]:
dataf_to_impute['Age'][dataf_to_impute['PassengerId'] == Id] = title_age.loc[tle].sum()
def title_maker(dataframe):
dataframe['Title'] =(dataframe['Name'][dataframe['Name'].notnull() ] ).apply(lambda x: list(list(x.split(',')) [1].split(' ')) [1] )
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 SAKT_model = SAKTModel(n_skill, embed_dim=128)
 try:
SAKT_model.load_state_dict(torch.load("/kaggle/input/riiid-sakt-model-dataset-public/sakt_model.pt"))
except:
SAKT_model.load_state_dict(torch.load("/kaggle/input/riiid-sakt-model-dataset-public/sakt_model.pt", map_location='cpu'))
SAKT_model.to(device)
 SAKT_model.eval()<split>
train_df['Age'].isna().sum()
def inference(TARGET, FEATURES, model, questions_df, prior_question_elapsed_time_mean, features_dicts):
answered_correctly_u_count = features_dicts['answered_correctly_u_count']
answered_correctly_u_sum = features_dicts['answered_correctly_u_sum']
elapsed_time_u_sum = features_dicts['elapsed_time_u_sum']
explanation_u_sum = features_dicts['explanation_u_sum']
answered_correctly_q_count = features_dicts['answered_correctly_q_count']
answered_correctly_q_sum = features_dicts['answered_correctly_q_sum']
elapsed_time_q_sum = features_dicts['elapsed_time_q_sum']
explanation_q_sum = features_dicts['explanation_q_sum']
answered_correctly_uq = features_dicts['answered_correctly_uq']
timestamp_u = features_dicts['timestamp_u']
timestamp_u_incorrect = features_dicts['timestamp_u_incorrect']
env = riiideducation.make_env()
iter_test = env.iter_test()
set_predict = env.predict
previous_test_df = None
for(test_df, sample_prediction_df)in iter_test:
if(previous_test_df is not None)&(psutil.virtual_memory().percent<90):
print(psutil.virtual_memory().percent)
 previous_test_df[TARGET] = eval(test_df["prior_group_answers_correct"].iloc[0])
 previous_test_df = previous_test_df[previous_test_df[TARGET] != -1].reset_index(drop=True)
 previous_test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 previous_test_df.prior_question_had_explanation=previous_test_df.prior_question_had_explanation.astype('int8')
 prev_group = previous_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
prev_group_content = prev_group[prev_user_id][0]
prev_group_ac = prev_group[prev_user_id][1]
if prev_user_id in group.index:
group[prev_user_id] =(np.append(group[prev_user_id][0],prev_group_content),
np.append(group[prev_user_id][1],prev_group_ac))
else:
group[prev_user_id] =(prev_group_content,prev_group_ac)
 if len(group[prev_user_id][0])>MAX_SEQ:
new_group_content = group[prev_user_id][0][-MAX_SEQ:]
new_group_ac = group[prev_user_id][1][-MAX_SEQ:]
group[prev_user_id] =(new_group_content,new_group_ac)
 update_features(previous_test_df, answered_correctly_u_sum, answered_correctly_q_sum, timestamp_u_incorrect)
 previous_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop = True)
 test_dataset = TestDataset(group, test_df, skills)
 test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)
 SAKT_outs = []
for item in test_dataloader:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = SAKT_model(x, target_id)
 output = torch.sigmoid(output)
 output = output[:, -1]
SAKT_outs.extend(output.view(-1 ).data.cpu().numpy())
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df.prior_question_had_explanation=test_df.prior_question_had_explanation.astype('int8')
 test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)
 test_df = pd.merge(test_df, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df[TARGET] = 0
test_df = add_features(test_df, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq, update = False)
 sub_preds = np.zeros(test_df.shape[0])
 for i, model in enumerate(clfs, 1):
test_preds = model.predict(test_df[FEATURES])
 sub_preds += test_preds
lgbm_final = sub_preds / len(clfs)
 test_df[TARGET] = np.array(SAKT_outs)* 0.4 + lgbm_final * 0.6
set_predict(test_df[['row_id', TARGET]])
 print('Thanks')
 inference(TARGET, FEATURES, model, questions_df, prior_question_elapsed_time_mean, features_dicts )<import_modules>
age_map = {'Unknown' : 0,
'Baby' : 1,
'Child' : 2,
'Student' : 3,
'Teenager' : 4,
'Young Adult' : 5,
'Adult' : 6,
'Senior' : 7}
train_df['AgeGroup'] = train_df['AgeGroup'].replace(age_map )
import pandas as pd
import numpy as np
import gc
import pickle
import psutil
import joblib
from sklearn.metrics import roc_auc_score
from collections import defaultdict
from tqdm.notebook import tqdm
import lightgbm as lgb
import riiideducation
import matplotlib.pyplot as plt
import seaborn as sns
import random
import os<define_variables>
Y = train_df['Survived'].values.ravel()
X_new = train_df[['Pclass','FareGroup','AgeGroup','SibSpBool','ParchBool','Sex','CabinBool']]
X_orig = train_df[['Pclass','Sex','Age','SibSp','Parch','Fare','CabinBool']]
TARGET = 'answered_correctly'
FEATURES = ['prior_question_elapsed_time',
'prior_question_had_explanation',
'content_field',
'answered_correctly_u_avg',
'elapsed_time_u_avg',
'explanation_u_avg',
'elapsed_time_q_avg',
'explanation_q_avg',
'explanation_qtrue_avg',
'explanation_qfalse_avg',
'beta_q',
'answered_correctly_uq_count',
'timestamp_u_recency_1',
'timestamp_u_recency_2',
'timestamp_u_recency_3',
'timestamp_u_incorrect_recency',
'theta_u',
'performance_u',
'task_container_avg',
'tags_avg',
'mean_question_accuracy',
'std_accuracy',
'tags_encoded',
'tag_1',
'tag_2',
'answered_correctly_u_num',
'answered_correctly_u_num_field',
'answered_correctly_u_avg_field',
'answered_correctly_difficulty_weighted_avg',
'answered_correctly_difficulty_weighted_avg_field',
'min_u_wrong_difficulty',
'min_u_wrong_difficulty_field',
'max_u_solved_difficulty',
'max_u_solved_difficulty_field',
'session_u_time',
'time_u_to_last_session',
'elo_rate',
'tags_lsi',
'question_elapsed_time_mean',
'tag_acc_max',
'tag_acc_min',
'tag_acc_count',
'explanation_qtrue_mean',
'explanation_qfalse_mean'
]<define_variables>
model_RF = RandomForestClassifier(random_state=0)
 my_pipeline = make_pipeline(model_RF)
 scores1_RF = cross_val_score(my_pipeline,X_orig,Y,scoring = 'accuracy',cv=5)
 scores2_RF = cross_val_score(my_pipeline,X_new,Y,scoring = 'accuracy', cv=5)
 Y_preds1 = cross_val_predict(my_pipeline,X_orig,Y)
 print('Score for model with original features : {}'.format(scores1_RF*100))
print('Score for model with simplified features : {}'.format(scores2_RF*100))
SEED = 123
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 seed_everything(SEED )<compute_test_metric>
model = XGBClassifier(random_state=0)
 my_pipeline = make_pipeline(model)
 scores1_XG = cross_val_score(my_pipeline,X_orig,Y,scoring = 'accuracy',cv=5)
 scores2_XG = cross_val_score(my_pipeline,X_new,Y,scoring = 'accuracy', cv=5)
 Y_preds1 = cross_val_predict(my_pipeline,X_orig,Y)
 print('Score for model with original features : {}'.format(scores1_XG*100))
print('Score for model with simplified features : {}'.format(scores2_XG*100))
def get_new_theta(is_good_answer, beta, theta, nb_previous_answers):
return theta + learning_rate_theta(nb_previous_answers)*(
 is_good_answer - probability_of_good_answer(theta, beta)
)
 def get_new_beta(is_good_answer, beta, theta, nb_previous_answers):
return beta - learning_rate_beta(nb_previous_answers)*(
 is_good_answer - probability_of_good_answer(theta, beta)
)
 def learning_rate_theta(nb_answers):
return max(0.3 /(1 + 0.01 * nb_answers), 0.04)
 def learning_rate_beta(nb_answers):
return 1 /(1 + 0.05 * nb_answers)
 def probability_of_good_answer(theta, beta, left_asymptote = 1/4):
return left_asymptote +(1 - left_asymptote)* sigmoid(theta - beta)
 def sigmoid(x):
return 1 /(1 + np.exp(-x))<feature_engineering>
model = SVC(random_state=0, gamma="auto")
 my_pipeline = make_pipeline(model)
 scores1_SV = cross_val_score(my_pipeline,X_orig,Y,scoring = 'accuracy',cv=5)
 scores2_SV = cross_val_score(my_pipeline,X_new,Y,scoring = 'accuracy', cv=5)
 Y_preds1 = cross_val_predict(my_pipeline,X_orig,Y)
 print('Score for model with original features : {}'.format(scores1_SV*100))
print('Score for model with simplified features : {}'.format(scores2_SV*100))
def add_train_features(df,
answered_correctly_u_count,
answered_correctly_u_sum,
elapsed_time_u_sum,
explanation_u_sum,
timestamp_u,
timestamp_u_incorrect,
latest_u_theta,
answered_correctly_q_count,
answered_correctly_q_sum,
elapsed_time_q_sum,
explanation_q_sum,
explanation_qtrue_sum,
explanation_qtrue_count,
latest_q_beta,
answered_correctly_uq,
question_avg_sum_u,
task_container_sum,
task_container_count,
tags_sum,
tags_count,
answered_correctly_u_count_field,
answered_correctly_u_sum_field,
answered_correctly_difficulty_weighted_sum,
answered_correctly_difficulty_weighted_sum_field,
max_solved_difficulty,
max_solved_difficulty_field,
min_wrong_difficulty,
min_wrong_difficulty_field,
session_time,
since_last_session_time,
last_session_start_time,
first_action_time,
update = True):
df['tags'] = df['tags'].\
apply(lambda ts: [int(x)for x in str(ts ).split() if x != 'nan'])
 for _, row in enumerate(df[['user_id',
'answered_correctly',
'content_id',
'prior_question_elapsed_time',
'prior_question_had_explanation',
'timestamp',
'task_container_id',
'tags',
'content_field',
'mean_question_accuracy',
]].values):
field = int(row[8])- 1
if row[0] not in answered_correctly_u_count_field:
answered_correctly_u_count_field[row[0]] = [0] * 7
answered_correctly_u_sum_field[row[0]] = [0] * 7
answered_correctly_difficulty_weighted_sum_field[row[0]] = [0] * 7
max_solved_difficulty_field[row[0]] = [0] * 7
min_wrong_difficulty_field[row[0]] = [0] * 7
if first_action_time[row[0]] == 0:
first_action_time[row[0]] = row[5]
last_session_start_time[row[0]] = row[5]
else:
if row[5] - last_session_start_time[row[0]] >= 7200 * 1000:
since_last_session_time[row[0]] =(row[5] - last_session_start_time[row[0]])/\
1000 / 3600
last_session_start_time[row[0]] = row[5]
answered_correctly_u_count[row[0]] += 1
answered_correctly_u_count_field[row[0]][field] += 1
elapsed_time_u_sum[row[0]] += row[3]
explanation_u_sum[row[0]] += int(row[4])
 if len(timestamp_u[row[0]])== 3:
timestamp_u[row[0]].pop(0)
 timestamp_u[row[0]].append(row[5])
 else:
timestamp_u[row[0]].append(row[5])
 answered_correctly_q_count[row[2]] += 1
elapsed_time_q_sum[row[2]] += row[3]
explanation_q_sum[row[2]] += int(row[4])
 if row[4]:
explanation_qtrue_count[row[0]] += 1
if row[0] not in answered_correctly_uq:
answered_correctly_uq[row[0]] = defaultdict(int)
 answered_correctly_uq[row[0]][row[2]] += 1
task_container_count[row[6]] += 1
tags = row[7]
for k in range(len(tags)) :
tags_count[tags[k]] += 1
if update:
theta = latest_u_theta[row[0]]
beta = latest_q_beta[row[2]]
answered_correctly_u_sum[row[0]] += row[1]
answered_correctly_u_sum_field[row[0]][field] += row[1]
answered_correctly_difficulty_weighted_sum[row[0]] += row[1] *(1 - row[9])* 3
answered_correctly_difficulty_weighted_sum_field[row[0]][field] += row[1] *(1 - row[9])* 3
if row[1] == 0:
if len(timestamp_u_incorrect[row[0]])== 1:
timestamp_u_incorrect[row[0]].pop(0)
 timestamp_u_incorrect[row[0]].append(row[5])
 else:
timestamp_u_incorrect[row[0]].append(row[5])
 if row[9] > min_wrong_difficulty[row[0]]:
min_wrong_difficulty[row[0]] = row[9]
if row[9] > min_wrong_difficulty_field[row[0]][field]:
min_wrong_difficulty_field[row[0]][field] = row[9]
else:
if 1 - row[9] > max_solved_difficulty[row[0]]:
max_solved_difficulty[row[0]] = 1 - row[9]
if 1 - row[9] > max_solved_difficulty_field[row[0]][field]:
max_solved_difficulty_field[row[0]][field] = 1 - row[9]
latest_u_theta[row[0]] = get_new_theta(row[1],
beta, theta,
answered_correctly_u_count[row[0]])
 task_container_sum[row[6]] += row[1]
for k in range(len(tags)) :
tags_sum[tags[k]] += row[1]
answered_correctly_q_sum[row[2]] += row[1]
if row[4]:
explanation_qtrue_sum[row[0]] += row[1]
latest_q_beta[row[2]] = get_new_beta(row[1],
beta, theta,
answered_correctly_q_count[row[2]])
 question_avg_sum_u[row[0]] += answered_correctly_q_sum[row[2]] /\
answered_correctly_q_count[row[2]]<feature_engineering>
test_df1 = test_df
test_df1['Age'].isna().sum()
def add_features(df,
answered_correctly_u_count,
answered_correctly_u_sum,
elapsed_time_u_sum,
explanation_u_sum,
timestamp_u,
timestamp_u_incorrect,
latest_u_theta,
answered_correctly_q_count,
answered_correctly_q_sum,
elapsed_time_q_sum,
explanation_q_sum,
explanation_qtrue_sum,
explanation_qtrue_count,
latest_q_beta,
answered_correctly_uq,
question_avg_sum_u,
task_container_sum,
task_container_count,
tags_sum,
tags_count,
answered_correctly_u_count_field,
answered_correctly_u_sum_field,
answered_correctly_difficulty_weighted_sum,
answered_correctly_difficulty_weighted_sum_field,
max_solved_difficulty,
max_solved_difficulty_field,
min_wrong_difficulty,
min_wrong_difficulty_field,
session_time,
since_last_session_time,
last_session_start_time,
first_action_time,
update = True):
answered_correctly_u_avg = np.zeros(len(df), dtype = np.float32)
 elapsed_time_u_avg = np.zeros(len(df), dtype = np.float32)
 explanation_u_avg = np.zeros(len(df), dtype = np.float32)
 timestamp_u_recency_1 = np.zeros(len(df), dtype = np.float32)
 timestamp_u_recency_2 = np.zeros(len(df), dtype = np.float32)
 timestamp_u_recency_3 = np.zeros(len(df), dtype = np.float32)
 timestamp_u_incorrect_recency = np.zeros(len(df), dtype = np.float32)
 theta_u = np.zeros(len(df), dtype = np.float32)
 answered_correctly_u_num = np.zeros(len(df), dtype = np.int32)
 answered_correctly_u_num_field = np.zeros(len(df), dtype = np.int32)
 answered_correctly_u_avg_field = np.zeros(len(df), dtype = np.float32)
 answered_correctly_difficulty_weighted_avg = np.zeros(len(df), dtype = np.float32)
 answered_correctly_difficulty_weighted_avg_field = np.zeros(len(df), dtype = np.float32)
 max_u_solved_difficulty = np.zeros(len(df), dtype = np.float32)
 max_u_solved_difficulty_field = np.zeros(len(df), dtype = np.float32)
 min_u_wrong_difficulty = np.zeros(len(df), dtype = np.float32)
 min_u_wrong_difficulty_field = np.zeros(len(df), dtype = np.float32)
 session_u_time = np.zeros(len(df), dtype = np.float32)
 time_u_to_last_session = np.zeros(len(df), dtype = np.float32)
 elapsed_time_q_avg = np.zeros(len(df), dtype = np.float32)
 explanation_q_avg = np.zeros(len(df), dtype = np.float32)
 explanation_qtrue_avg = np.zeros(len(df), dtype = np.float32)
 explanation_qfalse_avg = np.zeros(len(df), dtype = np.float32)
 beta_q = np.zeros(len(df), dtype = np.float32)
 answered_correctly_uq_count = np.zeros(len(df), dtype = np.int32)
 performance_u = np.zeros(len(df), dtype = np.float32)
 elo_rate = np.zeros(len(df), dtype = np.float32)
 task_container_avg = np.zeros(len(df), dtype = np.float32)
 tags_avg = np.zeros(len(df), dtype = np.float32)
 for num, row in enumerate(df[['user_id',
'answered_correctly',
'content_id',
'prior_question_elapsed_time',
'prior_question_had_explanation',
'timestamp',
'task_container_id',
'tags',
'content_field',
'mean_question_accuracy',
]].values):
field = int(row[8])- 1
answered_correctly_u_num[num] = answered_correctly_u_count[row[0]]
if answered_correctly_u_count[row[0]] != 0:
answered_correctly_u_avg[num] = answered_correctly_u_sum[row[0]] /\
answered_correctly_u_count[row[0]]
elapsed_time_u_avg[num] = elapsed_time_u_sum[row[0]] /\
answered_correctly_u_count[row[0]]
explanation_u_avg[num] = explanation_u_sum[row[0]] /\
answered_correctly_u_count[row[0]]
performance_u[num] = answered_correctly_u_avg[num] - \
question_avg_sum_u[row[0]] / answered_correctly_u_count[row[0]]
theta_u[num] = latest_u_theta[row[0]]
answered_correctly_difficulty_weighted_avg[num] = answered_correctly_difficulty_weighted_sum[row[0]] /\
answered_correctly_u_count[row[0]]
max_u_solved_difficulty[num] = max_solved_difficulty[row[0]]
min_u_wrong_difficulty[num] = min_wrong_difficulty[row[0]]
else:
answered_correctly_u_avg[num] = np.nan
elapsed_time_u_avg[num] = np.nan
explanation_u_avg[num] = np.nan
performance_u[num] = np.nan
theta_u[num] = 0
answered_correctly_difficulty_weighted_avg[num] = np.nan
max_u_solved_difficulty[num] = np.nan
min_u_wrong_difficulty[num] = np.nan
if row[0] not in answered_correctly_u_count_field:
answered_correctly_u_count_field[row[0]] = [0] * 7
answered_correctly_u_sum_field[row[0]] = [0] * 7
answered_correctly_difficulty_weighted_sum_field[row[0]] = [0] * 7
max_solved_difficulty_field[row[0]] = [0] * 7
min_wrong_difficulty_field[row[0]] = [0] * 7
answered_correctly_u_num_field[num] = answered_correctly_u_count_field[row[0]][field]
if answered_correctly_u_count_field[row[0]][field] != 0:
answered_correctly_u_avg_field[num] = answered_correctly_u_sum_field[row[0]][field] /\
answered_correctly_u_count_field[row[0]][field]
answered_correctly_difficulty_weighted_avg_field[num] = answered_correctly_difficulty_weighted_sum_field[row[0]][field] /\
answered_correctly_u_count_field[row[0]][field]
max_u_solved_difficulty_field[num] = max_solved_difficulty_field[row[0]][field]
min_u_wrong_difficulty_field[num] = min_wrong_difficulty_field[row[0]][field]
else:
answered_correctly_u_avg_field[num] = np.nan
answered_correctly_difficulty_weighted_avg_field[num] = np.nan
max_u_solved_difficulty_field[num] = np.nan
min_u_wrong_difficulty_field[num] = np.nan
if first_action_time[row[0]] == 0:
first_action_time[row[0]] = row[5]
last_session_start_time[row[0]] = row[5]
else:
if row[5] - last_session_start_time[row[0]] >= 7200 * 1000:
since_last_session_time[row[0]] =(row[5] - last_session_start_time[row[0]])/\
1000 / 3600
last_session_start_time[row[0]] = row[5]
session_u_time[num] = 0
else:
session_u_time[num] =(row[5] - last_session_start_time[row[0]])/\
1000 / 60
time_u_to_last_session[num] = since_last_session_time[row[0]]
if len(timestamp_u[row[0]])== 0:
timestamp_u_recency_1[num] = np.nan
timestamp_u_recency_2[num] = np.nan
timestamp_u_recency_3[num] = np.nan
elif len(timestamp_u[row[0]])== 1:
timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][0]
timestamp_u_recency_2[num] = np.nan
timestamp_u_recency_3[num] = np.nan
elif len(timestamp_u[row[0]])== 2:
timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][1]
timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][0]
timestamp_u_recency_3[num] = np.nan
elif len(timestamp_u[row[0]])== 3:
timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][2]
timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][1]
timestamp_u_recency_3[num] = row[5] - timestamp_u[row[0]][0]
if len(timestamp_u_incorrect[row[0]])== 0:
timestamp_u_incorrect_recency[num] = np.nan
else:
timestamp_u_incorrect_recency[num] = row[5] - timestamp_u_incorrect[row[0]][0]
if answered_correctly_q_count[row[2]] != 0:
elapsed_time_q_avg[num] = elapsed_time_q_sum[row[2]] /\
answered_correctly_q_count[row[2]]
explanation_q_avg[num] = explanation_q_sum[row[2]] /\
answered_correctly_q_count[row[2]]
beta_q[num] = latest_q_beta[row[2]]
else:
elapsed_time_q_avg[num] = np.nan
explanation_q_avg[num] = np.nan
beta_q[num] = 0
if explanation_qtrue_count[row[0]] != 0:
explanation_qtrue_avg[num] = explanation_qtrue_sum[row[0]] /\
explanation_qtrue_count[row[0]]
else:
explanation_qtrue_avg[num] = np.nan
if answered_correctly_u_count[row[0]] - explanation_qtrue_count[row[0]] != 0:
explanation_qfalse_avg[num] = \ 
(answered_correctly_u_sum[row[0]] - explanation_qtrue_sum[row[0]])/\ 
(answered_correctly_u_count[row[0]] - explanation_qtrue_count[row[0]])
 else:
explanation_qfalse_avg[num] = np.nan
elo_rate[num] = latest_u_theta[row[0]] - latest_q_beta[row[2]]
if task_container_count[row[6]] != 0:
task_container_avg[num] = task_container_sum[row[6]] /\
task_container_count[row[6]]
else:
task_container_avg[num] = np.nan
tags = row[7]
tags_means = []
for k in range(len(tags)) :
if tags_count[tags[k]] == 0:
continue
tags_means.append(tags_sum[tags[k]] / tags_count[tags[k]])
 if not tags_means:
tags_avg[num] = np.nan
else:
tags_avg[num] = np.mean(tags_means)
 answered_correctly_u_count[row[0]] += 1
answered_correctly_u_count_field[row[0]][field] += 1
elapsed_time_u_sum[row[0]] += row[3]
explanation_u_sum[row[0]] += int(row[4])
 if len(timestamp_u[row[0]])== 3:
timestamp_u[row[0]].pop(0)
 timestamp_u[row[0]].append(row[5])
 else:
timestamp_u[row[0]].append(row[5])
 answered_correctly_q_count[row[2]] += 1
elapsed_time_q_sum[row[2]] += row[3]
explanation_q_sum[row[2]] += int(row[4])
 if row[4]:
explanation_qtrue_count[row[0]] += 1
if row[0] not in answered_correctly_uq:
answered_correctly_uq[row[0]] = defaultdict(int)
 answered_correctly_uq_count[num] = answered_correctly_uq[row[0]][row[2]]
answered_correctly_uq[row[0]][row[2]] += 1
task_container_count[row[6]] += 1
for k in range(len(tags)) :
tags_count[tags[k]] += 1
if update:
theta = latest_u_theta[row[0]]
beta = latest_q_beta[row[2]]
answered_correctly_u_sum[row[0]] += row[1]
answered_correctly_u_sum_field[row[0]][field] += row[1]
answered_correctly_difficulty_weighted_sum[row[0]] += row[1] *(1 - row[9])* 3
answered_correctly_difficulty_weighted_sum_field[row[0]][field] += row[1] *(1 - row[9])* 3
if row[1] == 0:
if len(timestamp_u_incorrect[row[0]])== 1:
timestamp_u_incorrect[row[0]].pop(0)
 timestamp_u_incorrect[row[0]].append(row[5])
 else:
timestamp_u_incorrect[row[0]].append(row[5])
 if row[9] > min_wrong_difficulty[row[0]]:
min_wrong_difficulty[row[0]] = row[9]
if row[9] > min_wrong_difficulty_field[row[0]][field]:
min_wrong_difficulty_field[row[0]][field] = row[9]
else:
if 1 - row[9] > max_solved_difficulty[row[0]]:
max_solved_difficulty[row[0]] = 1 - row[9]
if 1 - row[9] > max_solved_difficulty_field[row[0]][field]:
max_solved_difficulty_field[row[0]][field] = 1 - row[9]
latest_u_theta[row[0]] = get_new_theta(row[1],
beta, theta,
answered_correctly_u_count[row[0]])
 task_container_sum[row[6]] += row[1]
for k in range(len(tags)) :
tags_sum[tags[k]] += row[1]
answered_correctly_q_sum[row[2]] += row[1]
if row[4]:
explanation_qtrue_sum[row[0]] += row[1]
latest_q_beta[row[2]] = get_new_beta(row[1],
beta, theta,
answered_correctly_q_count[row[2]])
 question_avg_sum_u[row[0]] += answered_correctly_q_sum[row[2]] /\
answered_correctly_q_count[row[2]]
user_df = pd.DataFrame({'answered_correctly_u_avg': answered_correctly_u_avg,
'elapsed_time_u_avg': elapsed_time_u_avg,
'explanation_u_avg': explanation_u_avg,
'elapsed_time_q_avg': elapsed_time_q_avg,
'explanation_q_avg': explanation_q_avg,
'explanation_qtrue_avg': explanation_qtrue_avg,
'explanation_qfalse_avg': explanation_qfalse_avg,
'answered_correctly_uq_count': answered_correctly_uq_count,
'timestamp_u_recency_1': timestamp_u_recency_1,
'timestamp_u_recency_2': timestamp_u_recency_2,
'timestamp_u_recency_3': timestamp_u_recency_3,
'timestamp_u_incorrect_recency': timestamp_u_incorrect_recency,
'performance_u': performance_u,
'tags_avg': tags_avg,
'answered_correctly_u_num': answered_correctly_u_num,
'answered_correctly_u_num_field': answered_correctly_u_num_field,
'answered_correctly_u_avg_field': answered_correctly_u_avg_field,
'answered_correctly_difficulty_weighted_avg': answered_correctly_difficulty_weighted_avg,
'answered_correctly_difficulty_weighted_avg_field': answered_correctly_difficulty_weighted_avg_field,
'max_u_solved_difficulty': max_u_solved_difficulty,
'max_u_solved_difficulty_field': max_u_solved_difficulty_field,
'min_u_wrong_difficulty': min_u_wrong_difficulty,
'min_u_wrong_difficulty_field': min_u_wrong_difficulty_field,
'session_u_time': session_u_time,
'time_u_to_last_session': time_u_to_last_session,
'answered_correctly_uq_count': answered_correctly_uq_count,
'theta_u': theta_u,
'beta_q': beta_q,
'elo_rate': elo_rate,
'task_container_avg': task_container_avg,
})
 df = pd.concat([df, user_df], axis = 1)
 return df<feature_engineering>
bins = [-1,0 ,5 ,12 , 18, 24, 35, 60, np.inf]
labels = ['Unknown','Baby','Child','Student','Teenager','Young Adult','Adult','Senior']
test_df1['AgeGroup'] = pd.cut(test_df1['Age'],bins,labels = labels)
 age_map = {'Unknown' : 0,
'Baby' : 1,
'Child' : 2,
'Student' : 3,
'Teenager' : 4,
'Young Adult' : 5,
'Adult' : 6,
'Senior' : 7}
test_df1['AgeGroup'] = test_df1['AgeGroup'].replace(age_map)
 sex_map = {'male' : 0, 'female' : 1}
test_df1['Sex'] = test_df1['Sex'].replace(sex_map)
 test_df1['CabinBool'] =(test_df1['Cabin'].notnull().astype('int'))
test_df1['Fare'] = test_df1['Fare'].fillna(-0.5)
 bins = [-1, 0, 20 ,41 , 60, 81, 100, np.inf]
labels = ['Unknown','1-20','21-41','42-60','61-81','82-100','101+']
test_df1['FareGroup'] = pd.cut(test_df1['Fare'],bins,labels = labels)
 fare_map = {'Unknown' : 0,'1-20' : 1,'21-41' : 2,'42-60' :3 ,'61-81' : 4,'82-100' : 5,'101+' : 6}
test_df1['FareGroup'] = test_df1['FareGroup'].replace(fare_map)
 test_df1['ParchBool'] =(test_df1['Parch'].apply(lambda x: 1 if x>0 else 0))
test_df1['SibSpBool'] =(test_df1['SibSp'].apply(lambda x: 1 if x>0 else 0))
def update_features(df,
answered_correctly_u_sum,
answered_correctly_q_sum,
timestamp_u_incorrect,
explanation_qtrue_sum,
task_container_sum,
latest_u_theta,
latest_q_beta,
answered_correctly_u_count,
answered_correctly_q_count,
tags_sum,
answered_correctly_u_sum_field,
answered_correctly_difficulty_weighted_sum,
answered_correctly_difficulty_weighted_sum_field,
max_solved_difficulty,
max_solved_difficulty_field,
min_wrong_difficulty,
min_wrong_difficulty_field, 
):
for row in df[['user_id',
'answered_correctly',
'content_id',
'content_type_id',
'timestamp',
'prior_question_had_explanation',
'task_container_id',
'tags',
'content_field',
'mean_question_accuracy',
]].values:
if row[3] == 0:
field = int(row[8])- 1
theta = latest_u_theta[row[0]]
beta = latest_q_beta[row[2]]
answered_correctly_u_sum[row[0]] += row[1]
answered_correctly_u_sum_field[row[0]][field] += row[1]
answered_correctly_difficulty_weighted_sum[row[0]] += row[1] *(1 - row[9])* 3
answered_correctly_difficulty_weighted_sum_field[row[0]][field] += row[1] *(1 - row[9])* 3
if row[1] == 0:
if len(timestamp_u_incorrect[row[0]])== 1:
timestamp_u_incorrect[row[0]].pop(0)
 timestamp_u_incorrect[row[0]].append(row[4])
 else:
timestamp_u_incorrect[row[0]].append(row[4])
 if row[9] > min_wrong_difficulty[row[0]]:
min_wrong_difficulty[row[0]] = row[9]
if row[9] > min_wrong_difficulty_field[row[0]][field]:
min_wrong_difficulty_field[row[0]][field] = row[9]
else:
if 1 - row[9] > max_solved_difficulty[row[0]]:
max_solved_difficulty[row[0]] = 1 - row[9]
if 1 - row[9] > max_solved_difficulty_field[row[0]][field]:
max_solved_difficulty_field[row[0]][field] = 1 - row[9]
latest_u_theta[row[0]] = get_new_theta(row[1],
beta, theta,
answered_correctly_u_count[row[0]])
 task_container_sum[row[6]] += row[1]
tags = row[7]
for k in range(len(tags)) :
tags_sum[tags[k]] += row[1]
answered_correctly_q_sum[row[2]] += row[1]
if row[5]:
explanation_qtrue_sum[row[0]] += row[1]
latest_q_beta[row[2]] = get_new_beta(row[1],
beta, theta,
answered_correctly_q_count[row[2]])
 return<load_from_csv>
ids = test_df1['PassengerId']
model = SVC(random_state=0, gamma="auto")
 model.fit(X_new,Y)
 test_df1_pred = test_df1[X_new.columns]
predictions = model.predict(test_df1_pred)
 output = pd.DataFrame({'PassengerId' : ids,
'Survived' : predictions})
 output.to_csv('submission.csv', index=False )