Kaggle competition: Riiid Answer Correctness Prediction
scaler_cols = ['Age', 'Fare', 'Name_Length', 'Family_Size', 'Name_Length',
'Ticket_Frequency', 'Fare_Family_Size', 'Fare_Cat_Pclass']
std = StandardScaler()
std.fit(df_train_final[scaler_cols])
 df_train_final.loc[:, scaler_cols] = pd.DataFrame(std.transform(df_train_final[scaler_cols]), index=df_train_final.index, columns = scaler_cols)
 df_test_final.loc[:, scaler_cols] = pd.DataFrame(std.transform(df_test_final[scaler_cols]), index=df_test_final.index, columns = scaler_cols )<define_variables>
full_train = pd.read_pickle(".. /input/train-gzip/riiid_train.gzip")
 questions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')
 lectures = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )
features = ['Pclass', 'Sex', 'Age', 'Fare', 'Title', 'Name_Length', 'Emb_C',
'Emb_Q', 'Emb_S', 'Title_Master', 'Title_Miss', 'Title_Mr', 'Title_Mrs',
'Title_Other', 'Title_Royal', 'Family_Size', 'Fsize_Cat',
'Family_Friends_Surv_Rate', 'Surv_Rate_Invalid', 'Cabin_Clean',
'Ticket_Frequency', 'Tkt_AS', 'Tkt_C', 'Tkt_CA',
'Tkt_CASOTON', 'Tkt_FC', 'Tkt_FCC', 'Tkt_Fa', 'Tkt_LINE',
'Tkt_NUM', 'Tkt_PC', 'Tkt_PP', 'Tkt_PPP', 'Tkt_SC', 'Tkt_SCA',
'Tkt_SCAH', 'Tkt_SCAHBasle', 'Tkt_SCOW', 'Tkt_SCPARIS', 'Tkt_SCParis',
'Tkt_SOC', 'Tkt_SOP', 'Tkt_SOPP', 'Tkt_SOTONO', 'Tkt_SOTONOQ', 'Tkt_SP',
'Tkt_STONO', 'Tkt_SWPP', 'Tkt_WC', 'Tkt_WEP', 'Fare_Cat',
'Fare_Family_Size', 'Fare_Cat_Pclass', 'Fare_Cat_Title',
'Fsize_Cat_Title', 'Fsize_Cat_Fare_Cat', 'Pclass_Title',
'Fsize_Cat_Pclass', 'Child', 'Senior']
features_train = ['Survived', 'Pclass', 'Sex', 'Age', 'Fare', 'Title', 'Name_Length', 'Emb_C',
'Emb_Q', 'Emb_S', 'Title_Master', 'Title_Miss', 'Title_Mr', 'Title_Mrs',
'Title_Other', 'Title_Royal', 'Family_Size', 'Fsize_Cat',
'Family_Friends_Surv_Rate', 'Surv_Rate_Invalid', 'Cabin_Clean',
'Ticket_Frequency', 'Tkt_AS', 'Tkt_C', 'Tkt_CA',
'Tkt_CASOTON', 'Tkt_FC', 'Tkt_FCC', 'Tkt_Fa', 'Tkt_LINE',
'Tkt_NUM', 'Tkt_PC', 'Tkt_PP', 'Tkt_PPP', 'Tkt_SC', 'Tkt_SCA',
'Tkt_SCAH', 'Tkt_SCAHBasle', 'Tkt_SCOW', 'Tkt_SCPARIS', 'Tkt_SCParis',
'Tkt_SOC', 'Tkt_SOP', 'Tkt_SOPP', 'Tkt_SOTONO', 'Tkt_SOTONOQ', 'Tkt_SP',
'Tkt_STONO', 'Tkt_SWPP', 'Tkt_WC', 'Tkt_WEP', 'Fare_Cat',
'Fare_Family_Size', 'Fare_Cat_Pclass', 'Fare_Cat_Title',
'Fsize_Cat_Title', 'Fsize_Cat_Fare_Cat', 'Pclass_Title',
'Fsize_Cat_Pclass', 'Child', 'Senior']
df_train_final = df_train_final[features_train]
df_test_final = df_test_final[features]<sort_values>
full_train = full_train[['row_id','user_id','content_id','content_type_id','answered_correctly']]
train = full_train.groupby('user_id' ).tail(500)
 test = full_train.groupby('user_id' ).tail(4)
 train = train.drop(test.index )
corr_mat = df_train_final.astype(float ).corr()
corr_mat_fil = corr_mat.loc[:, 'Survived'].sort_values(ascending=False)
 corr_mat_fil = pd.DataFrame(data=corr_mat_fil[1:] )<prepare_x_and_y>
question_average = pd.DataFrame(full_train.loc[full_train['content_type_id'] == 0].groupby(['content_id'])['answered_correctly'].mean() ).rename(columns={'answered_correctly':'question_average'})
 question_count = pd.DataFrame(full_train.loc[full_train['content_type_id'] == 0].groupby(['content_id'] ).size() ,columns=['question_count'])
 question_df = question_average.join(question_count)
 question_df['question_sum'] = question_df['question_average'] * question_df['question_count']
question_df = question_df.join(questions,how='outer')[['question_average','question_count','question_sum']]
question_df['question_average'].fillna(0,inplace=True)
 question_df['question_count'].fillna(0,inplace=True)
 question_df['question_sum'].fillna(0,inplace=True)
 del question_average,question_count
gc.collect()
features = df_test_final.columns.to_list()
X_train = df_train_final[features]
Y_train = df_train_final['Survived']
X_test = df_test_final<import_modules>
del full_train
gc.collect()
from sklearn.model_selection import cross_val_predict, cross_val_score, cross_validate
from sklearn.model_selection import StratifiedKFold, StratifiedShuffleSplit
from sklearn.metrics import confusion_matrix, roc_curve
from sklearn.metrics import precision_score, recall_score, f1_score
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.wrappers.scikit_learn import KerasClassifier
from tensorflow.keras.models import Sequential
from tensorflow.keras.utils import plot_model
from tensorflow.keras.layers import Input, Dense, Dropout, AlphaDropout, BatchNormalization,Concatenate, concatenate
from tensorflow.keras.optimizers import SGD, RMSprop, Adamax, Adagrad, Adam, Nadam, SGD
from tensorflow.keras.regularizers import l2
from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping
from tensorflow.keras.metrics import *<define_variables>
train = train.join(question_df,on=['content_id'], rsuffix='_question' )
metrics = ['accuracy',
Precision() ,
Recall() ]<choose_model_class>
mean_user = train.loc[train['content_type_id'] == False].groupby(['user_id'])['answered_correctly'].mean().mean()
mean_question = train.loc[train['content_type_id'] == False].groupby(['content_id'])['answered_correctly'].mean().mean()
def create_model() :
model = Sequential()
model.add(Input(shape=X_train.shape[1], name='Input_'))
model.add(Dense(8, activation='relu', kernel_initializer='glorot_normal', kernel_regularizer=l2(0.001)))
 model.add(Dense(16, activation='relu', kernel_initializer='glorot_normal', kernel_regularizer=l2(0.1)))
 model.add(Dropout(0.5))
model.add(Dense(16, activation='relu', kernel_initializer='glorot_normal', kernel_regularizer=l2(0.1)))
 model.add(Dropout(0.5))
model.add(Dense(1, activation='sigmoid', kernel_initializer='glorot_normal'))
model.summary()
optimize = Adam(lr = 0.0001)
 model.compile(optimizer = optimize,
loss = 'binary_crossentropy',
metrics = metrics)
 return model<compute_train_metric>
train['user_shift_question'] = train.loc[train['content_type_id'] == False].groupby(['user_id'])['question_average'].shift()
cumulated_question = train.loc[train['content_type_id'] == False].groupby(['user_id'])['user_shift_question'].agg(['cumsum','cumcount'])
 train.loc[train['content_type_id'] == False,'average_past_questions'] = cumulated_question['cumsum'] / cumulated_question['cumcount']
train['average_past_questions'].fillna(mean_question,inplace=True)
 train.drop(['user_shift_question','question_count','question_sum'],axis=1,inplace=True)
 del cumulated_question
estimator = KerasClassifier(build_fn = create_model, epochs = 600, batch_size = 32, verbose = 1)
 kfold = StratifiedKFold(n_splits = 3)
 results = cross_val_score(estimator, X_train, Y_train, cv = kfold )<train_model>
train['user_shift'] = train.loc[train['content_type_id'] == False].groupby(['user_id'])['answered_correctly'].shift()
cumulated = train.loc[train['content_type_id'] == False].groupby(['user_id'])['user_shift'].agg(['cumsum', 'cumcount'])
 train.loc[train['content_type_id'] == False,'answered_correctly_user_average'] = cumulated['cumsum'] / cumulated['cumcount']
train['answered_correctly_user_average'].fillna(mean_user,inplace=True)
 train.drop(columns=['user_shift'], inplace=True)
 del cumulated
train_history = estimator.fit(X_train, Y_train, epochs = 600, batch_size = 32 )<save_to_csv>
user_average = pd.DataFrame(train.loc[train['content_type_id'] == 0].groupby(['user_id'])['answered_correctly_user_average'].last() ).rename(columns={'answered_correctly_user_average':'user_average'})
 user_count = pd.DataFrame(train.loc[train['content_type_id'] == 0].groupby(['user_id'] ).size() - 1,columns=['user_count'] )
y_preds = estimator.predict(X_test)
 submission = pd.read_csv(".. /input/titanic/gender_submission.csv", index_col='PassengerId')
 submission['Survived'] = y_preds.astype(int)
 submission.to_csv('submission.csv' )<set_options>
tmp = train.loc[train['content_type_id'] == False].groupby(['user_id'] ).mean()
user_performance = pd.DataFrame(tmp['answered_correctly'] - tmp['question_average'], columns=['performance'])
 del tmp
%matplotlib inline
sns.set_style('dark')
 sns.set_palette('Set2')
<load_from_csv>
user_df = user_performance.join(user_average ).join(user_count)
 user_df['user_sum'] = user_df['user_average'] * user_df['user_count']
del user_performance, user_count, user_average
gc.collect()
train_data = pd.read_csv('.. /input/titanic/train.csv')
 test_data = pd.read_csv('.. /input/titanic/test.csv')
 data = pd.concat([train_data, test_data] ).reset_index().drop(['index'], axis=1 )<feature_engineering>
def question_average_sum_by_user(df,question_df):
my_dict = {}
group = df.groupby(['user_id'])
 for user, val in group:
average_sum = 0.0
for row_index, row in val.iterrows() :
if(row['content_type_id'] == False):
question_id = row['content_id']
question_average = question_df.at[question_id,'question_average']
average_sum += question_average
my_dict[user] = [average_sum]
return pd.DataFrame.from_dict(my_dict,orient='index',columns=['question_average_sum'] )
data['Surname'] = data['Name'].apply(lambda x: x.split(',')[0] )<feature_engineering>
def add_answers_to_prior_df(current_df,prior_df):
prior_df_ = prior_df.copy()
if(prior_df.shape[0] > 0):
val = eval(current_df.iloc[0]['prior_group_answers_correct'])
 if(len(val)== prior_df.shape[0]):
prior_df_['answered_correctly_response'] = val
return prior_df_
data['Ticket_id'] = 'new_col'
def ticket_id(row):
row['Ticket_id'] = str(row.Pclass)+ '-' + str(row.Ticket)[:-1] + '-' + str(row.Fare)+ '-' + str(row.Embarked)
 return row
data = data.apply(ticket_id, axis='columns' )<feature_engineering>
def build_question_df(prior_df,question_df):
if(prior_df.shape[0] == 0):
return question_df
question_sum_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['content_id'])['answered_correctly_response'].sum())\ 
.rename(columns={'answered_correctly_response':'question_sum'})
 question_count_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['content_id'] ).size() ,columns=['question_count'])
 question_df = question_df.join(question_sum_prior,rsuffix='_previous' ).join(question_count_prior,rsuffix='_previous')
 question_df['question_average'].fillna(0,inplace=True)
 question_df['question_count'].fillna(0,inplace=True)
 question_df['question_sum'].fillna(0,inplace=True)
 question_df['question_sum_previous'].fillna(0,inplace=True)
 question_df['question_count_previous'].fillna(0,inplace=True)
 question_df['question_sum'] = question_df['question_sum'] + question_df['question_sum_previous']
question_df['question_count'] = question_df['question_count'] + question_df['question_count_previous']
question_df['question_average'] = question_df['question_sum'] / question_df['question_count']
question_df.drop(['question_count_previous','question_sum_previous'],inplace=True,axis=1)
 return question_df
data['Group_id'] = 'new_col2'
def group_id(row):
row['Group_id'] = str(row.Surname)+ '-' + str(row.Ticket_id)
 return row
data = data.apply(group_id, axis='columns' )<feature_engineering>
def build_user_df(prior_df,user_df,question_df):
if(prior_df.shape[0] == 0):
return user_df
user_sum_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['user_id'])['answered_correctly_response'].sum())\ 
.rename(columns={'answered_correctly_response':'user_sum'})
 user_count_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['user_id'] ).size() ,columns=['user_count'])
 user_df = user_df.join(user_sum_prior,how='outer',rsuffix='_previous' ).join(user_count_prior,rsuffix='_previous')
 user_df['performance'].fillna(0,inplace=True)
 user_df['user_average'].fillna(0,inplace=True)
 user_df['user_count'].fillna(0,inplace=True)
 user_df['user_sum'].fillna(0,inplace=True)
 user_df['user_count_previous'].fillna(0,inplace=True)
 user_df['user_sum_previous'].fillna(0,inplace=True)
 user_df = user_df.join(question_average_sum_by_user(prior_df,question_df))
user_df['question_average_sum'].fillna(0,inplace=True)
 user_df['user_mean_performance'] =(user_df['user_sum'] - user_df['performance'] * user_df['user_count'] + user_df['question_average_sum'])/(user_df['user_count'] + user_df['user_count_previous'])
 user_df['user_sum'] = user_df['user_sum'] + user_df['user_sum_previous']
user_df['user_count'] = user_df['user_count'] + user_df['user_count_previous']
user_df['user_average'] = user_df['user_sum'] / user_df['user_count']
user_df['performance'] = user_df['user_average'] - user_df['user_mean_performance']
user_df.drop(['user_sum_previous','user_count_previous','question_average_sum','user_mean_performance'],axis=1,inplace=True)
 return user_df
data['Title'] = 'man'
data.loc[data.Sex == 'female', 'Title'] = 'woman'
data.loc[data['Name'].str.contains('Master'), 'Title'] = 'boy'<categorify>
prior_df = pd.DataFrame()
current_df = pd.DataFrame()
prior_df = add_answers_to_prior_df(current_df,prior_df)
 question_df = build_question_df(prior_df,question_df)
 user_df = build_user_df(prior_df,user_df,question_df )
data.loc[data.Title == 'man', 'Group_id'] = 'noGroup'
data['WC_count'] = data.loc[data.Title != 'man'].groupby('Group_id')['Group_id'].transform('count')
 data.loc[data.WC_count <=1, 'Group_id'] = 'noGroup'<filter>
TARGET_COL = ['answered_correctly']
FEATURE_COLS = ['row_id', 'performance', 'question_average']
cols = ['PassengerId', 'Survived', 'Name', 'Title', 'Ticket_id','Group_id']
data.loc[(data.Ticket_id == '1-1696-134.5-C')&(data.Title != 'man'), cols]<feature_engineering>
train = train[['row_id','user_id','content_id', 'content_type_id', 'answered_correctly']].reset_index()
indices = []
count = 0
for i in range(0,1309):
if(data.loc[i,'Title'] != 'man')&(data.loc[i,'Group_id'] == 'noGroup'):
data.loc[i,'Group_id'] = data.loc[(data['Ticket_id'] == data.loc[i, 'Ticket_id'])&(data.Title != 'man'), 'Group_id'].iloc[0]
if(data.loc[i, 'Group_id'] != 'noGroup'):
indices.append(i)
 count += 1
print('{:d} passengers were added to an existing group'.format(count))<filter>
def data_transform(df, is_training = True, is_validation = True):
df = df.join(question_df['question_average'],on=['content_id'],rsuffix='_question_average')
 df = df.join(user_df[['performance','user_average', 'user_count']],on=['user_id'],rsuffix='_right')
 df = df.loc[df['content_type_id'] == False]
if is_training or is_validation:
df = df[FEATURE_COLS + TARGET_COL]
else:
df = df[FEATURE_COLS]
return df
cols = ['PassengerId', 'Survived', 'Name', 'Title', 'Group_id']
data.loc[indices, cols]<count_unique_values>
%%time
train = data_transform(train )
number_of_groups = data.loc[data.Group_id != 'noGroup', 'Group_id'].nunique()
print('Number of groups found: {:d}'.format(number_of_groups))
number_of_WCG_passengers = data.loc[data.Group_id != 'noGroup', 'Group_id'].count()
print('
Number of passengers in a group: {:d}'.format(number_of_WCG_passengers))
composition = data.loc[data.Group_id != 'noGroup','Title'].value_counts()
print('
Composition of the groups:')
 print(composition.to_string() )<groupby>
%%time
test = data_transform(test,False,True )
data['WCSurvived'] = data.loc[(data.Title != 'man')&(data.Group_id != 'noGroup')].groupby('Group_id' ).Survived.transform('mean' )<filter>
X_train = train[FEATURE_COLS]
y_train = train[TARGET_COL]
X_test = test[FEATURE_COLS]
y_test = test[TARGET_COL]
cols = ['PassengerId', 'Survived', 'WCSurvived', 'Name', 'Title', 'Group_id']
data.loc[data.Group_id == 'Sage-3-CA.234-69.55-S', cols]<sort_values>
params = {
'objective': 'binary',
'seed': 42,
'metric': 'auc',
'learning_rate': 0.025,
'max_bin': 1000,
'num_leaves': 80,
'num_iterations' : 100
}
lgb_train = lgb.Dataset(X_train.iloc[:,1:],y_train)
 lgb_val = lgb.Dataset(X_test.iloc[:,1:],y_test )
data.loc[(data.WCSurvived==0.75)|(data.WCSurvived==0.5), cols].sort_values(by='Group_id' )<feature_engineering>
del X_train,y_train
gc.collect()
data.loc[data.Group_id.isin(test_groups), 'WCSurvived'] = 0
data.loc[(data.Group_id.isin(test_groups)) &(data.Pclass != 3), 'WCSurvived'] = 1<count_values>
model = lgb.train(
 params,
lgb_train,
valid_sets=[lgb_train,lgb_val],
verbose_eval=1,
num_boost_round=100,
early_stopping_rounds=10
)
print('WCSurvived test values:')
 print(data[891:1309].WCSurvived.value_counts().to_string() )<feature_engineering>
%%time
predictions = pd.DataFrame(model.predict(X_test.iloc[:,1:]),index=X_test.index )
data.loc[891:1308, 'Predict'] = 0
data.loc[891:1308, 'Predict'][(data.Sex == 'female')] = 1
data.loc[891:1308,'Predict'][(data.Sex == 'female')&(data['WCSurvived'] == 0)] = 0
data.loc[891:1308, 'Predict'][(data.Title == 'boy')&(data['WCSurvived'] == 1)] = 1 
<filter>
roc_auc_score(y_test,predictions[0] )
<filter><EOS> 
%%time
env = riiideducation.make_env()
iter_test = env.iter_test()
iter_nb = 0
for(current_df, sample_prediction_df)in iter_test:
if(iter_nb != 0):
prior_df = add_answers_to_prior_df(current_df,prior_df)
 question_df = build_question_df(prior_df,question_df)
 user_df = build_user_df(prior_df,user_df,question_df)
 prior_df = current_df.copy()
current_df = data_transform(current_df,False,False)
 current_df['answered_correctly'] = model.predict(current_df.iloc[:,1:])
 iter_nb = 1
env.predict(current_df.loc[:, ['row_id', 'answered_correctly']] )