Kaggle competition: Riiid Answer Correctness Prediction
train_and_test(nn, train_df.copy() , it = 10 )<set_options>
full_train = pd.read_pickle(".. /input/riiid-train-data-multiple-formats/riiid_train.pkl.gzip")
 questions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')
 lectures = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )
%matplotlib inline
py.init_notebook_mode(connected=True)
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 PassengerId = test['PassengerId']
train.head(3)
<feature_engineering>
full_train = full_train[['row_id','user_id','content_id','content_type_id','answered_correctly']]
train = full_train.groupby('user_id' ).tail(400)
 test = full_train.groupby('user_id' ).tail(4)
 train = train.drop(test.index )
original_train = train.copy()
full_data = [train, test]
train['Has_Cabin'] = train["Cabin"].apply(lambda x: 0 if type(x)== float else 1)
 test['Has_Cabin'] = test["Cabin"].apply(lambda x: 0 if type(x)== float else 1)
 for dataset in full_data:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
for dataset in full_data:
dataset['IsAlone'] = 0
dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1
for dataset in full_data:
dataset['Embarked'] = dataset['Embarked'].fillna('S')
 for dataset in full_data:
dataset['Fare'] = dataset['Fare'].fillna(train['Fare'].median())
 for dataset in full_data:
age_avg = dataset['Age'].mean()
age_std = dataset['Age'].std()
age_null_count = dataset['Age'].isnull().sum()
age_null_random_list = np.random.randint(age_avg - age_std, age_avg + age_std, size=age_null_count)
 dataset.loc[np.isnan(dataset['Age']), 'Age'] = age_null_random_list
dataset['Age'] = dataset['Age'].astype(int)
<feature_engineering>
question_average = pd.DataFrame(full_train.loc[full_train['content_type_id'] == 0].groupby(['content_id'])['answered_correctly'].mean() ).rename(columns={'answered_correctly':'question_average'})
 question_count = pd.DataFrame(full_train.loc[full_train['content_type_id'] == 0].groupby(['content_id'] ).size() ,columns=['question_count'])
 question_df = question_average.join(question_count)
 question_df['question_sum'] = question_df['question_average'] * question_df['question_count']
question_df = question_df.join(questions,how='outer')[['question_average','question_count','question_sum']]
question_df['question_average'].fillna(0,inplace=True)
 question_df['question_count'].fillna(0,inplace=True)
 question_df['question_sum'].fillna(0,inplace=True)
 del question_average,question_count
gc.collect()
def get_title(name):
title_search = re.search('([A-Za-z]+)\.', name)
 if title_search:
return title_search.group(1)
 return ""
for dataset in full_data:
dataset['Title'] = dataset['Name'].apply(get_title)
 for dataset in full_data:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 for dataset in full_data:
dataset['Sex'] = dataset['Sex'].map({'female': 0, 'male': 1} ).astype(int)
 title_mapping = {"Mr": 1, "Master": 2, "Mrs": 3, "Miss": 4, "Rare": 5}
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0)
 dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] 						 = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] 							 = 3
dataset['Fare'] = dataset['Fare'].astype(int)
 dataset.loc[ dataset['Age'] <= 16, 'Age'] 					 = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age'] ; 
<drop_column>
train = train.join(question_df,on=['content_id'], rsuffix='_question' )
drop_elements = ['PassengerId', 'Name', 'Ticket', 'Cabin', 'SibSp']
train = train.drop(drop_elements, axis = 1)
 test = test.drop(drop_elements, axis = 1 )<define_variables>
del full_train
gc.collect()
train[['Title', 'Survived']].groupby(['Title'], as_index=False ).agg(['mean', 'count', 'sum'])
<groupby>
def reduce_memory_usage(train_data):
train_data['prior_question_had_explanation'] = train_data['prior_question_had_explanation'].fillna(False ).astype('bool')
 return train_data
train[['Sex', 'Survived']].groupby(['Sex'], as_index=False ).agg(['mean', 'count', 'sum'])
<data_type_conversions>
mean_user = train.loc[train['content_type_id'] == False].groupby(['user_id'])['answered_correctly'].mean().mean()
mean_question = train.loc[train['content_type_id'] == False].groupby(['content_id'])['answered_correctly'].mean().mean()
title_and_sex = original_train.copy() [['Name', 'Sex']]
title_and_sex['Title'] = title_and_sex['Name'].apply(get_title)
 title_and_sex['Sex'] = title_and_sex['Sex'].map({'female': 0, 'male': 1} ).astype(int)
 title_and_sex[['Title', 'Sex']].groupby(['Title'], as_index=False ).agg(['mean', 'count', 'sum'])
<compute_train_metric>
train['user_shift_question'] = train.loc[train['content_type_id'] == False].groupby(['user_id'])['question_average'].shift()
def get_gini_impurity(survived_count, total_count):
survival_prob = survived_count/total_count
not_survival_prob =(1- survival_prob)
 random_observation_survived_prob = survival_prob
random_observation_not_survived_prob =(1 - random_observation_survived_prob)
 mislabelling_survided_prob = not_survival_prob * random_observation_survived_prob
mislabelling_not_survided_prob = survival_prob * random_observation_not_survived_prob
gini_impurity = mislabelling_survided_prob + mislabelling_not_survided_prob
return gini_impurity<choose_model_class>
train['user_shift_question'] = train.loc[train['content_type_id'] == False].groupby(['user_id'])['question_average'].shift()
cumulated_question = train.loc[train['content_type_id'] == False].groupby(['user_id'])['user_shift_question'].agg(['cumsum','cumcount'])
 train.loc[train['content_type_id'] == False,'average_past_questions'] = cumulated_question['cumsum'] /(cumulated_question['cumcount'] + 1)
 train.drop(columns=['user_shift_question'],inplace=True)
 train['average_past_questions'].fillna(mean_question,inplace=True)
 del cumulated_question
gini_impurity_starting_node = get_gini_impurity(342,891)
 gini_impurity_starting_node<find_best_params>
train['user_shift'] = train.loc[train['content_type_id'] == False].groupby(['user_id'])['answered_correctly'].shift()
cumulated = train.loc[train['content_type_id'] == False].groupby(['user_id'])['user_shift'].agg(['cumsum', 'cumcount'])
 train.loc[train['content_type_id'] == False,'answered_correctly_user_average'] = cumulated['cumsum'] / cumulated['cumcount']
train['answered_correctly_user_average'].fillna(mean_user,inplace=True)
 train.drop(columns=['user_shift'], inplace=True)
 del cumulated
gini_impurity_men = get_gini_impurity(109, 577)
 print(gini_impurity_men)
 gini_impurity_women = get_gini_impurity(233, 314)
 print(gini_impurity_women )<find_best_params>
user_average = pd.DataFrame(train.loc[train['content_type_id'] == 0].groupby(['user_id'])['answered_correctly_user_average'].last() ).rename(columns={'answered_correctly_user_average':'user_average'})
 user_count = pd.DataFrame(train.loc[train['content_type_id'] == 0].groupby(['user_id'] ).size() - 1,columns=['user_count'] )
gini_impurity_title_Mr = get_gini_impurity(81, 517)
 print(gini_impurity_title_Mr)
 gini_impurity_title_others = get_gini_impurity(261, 374)
 print(gini_impurity_title_others )<define_variables>
tmp = train.loc[train['content_type_id'] == False].groupby(['user_id'] ).mean()
train['performance_before'] = train['answered_correctly_user_average'] - train['average_past_questions']
user_performance = pd.DataFrame(train.loc[train['content_type_id'] == 0].groupby(['user_id'])['performance_before'].last() ).rename(columns={'performance_before':'performance'})
 del tmp
men_weight = 577/891
women_weight = 314/891
weighted_gini_impurity_sex_split =(gini_impurity_men * men_weight)+(gini_impurity_women * women_weight)
 print(weighted_gini_impurity_sex_split)
 title_1_weight = 517/891
title_others_weight = 374/891
weighted_gini_impurity_title_split =(gini_impurity_title_Mr * title_1_weight)+(gini_impurity_title_others * title_others_weight)
 print(weighted_gini_impurity_title_split )<define_variables>
gc.collect()
sex_gini_decrease = weighted_gini_impurity_sex_split - gini_impurity_starting_node
print(sex_gini_decrease)
 title_gini_decrease = weighted_gini_impurity_title_split - gini_impurity_starting_node
print(title_gini_decrease)
<choose_model_class>
user_df = user_performance.join(user_average ).join(user_count)
 user_df['user_sum'] = user_df['user_average'] * user_df['user_count']
cv = KFold(n_splits = 10)
 accuracies = list()
max_attributes = len(list(test))
depth_range = range(1, max_attributes + 1)
 for depth in depth_range:
fold_accuracy = []
tree_model = tree.DecisionTreeClassifier(max_depth = depth)
 for train_fold, valid_fold in cv.split(train):
f_train = train.loc[train_fold]
f_valid = train.loc[valid_fold]
model = tree_model.fit(X = f_train.drop(['Survived'], axis=1), y = f_train["Survived"])
 valid_acc = model.score(X = f_valid.drop(['Survived'], axis=1), y = f_valid["Survived"])
 fold_accuracy.append(valid_acc)
 avg = sum(fold_accuracy)/len(fold_accuracy)
 accuracies.append(avg)
 df = pd.DataFrame({"Max Depth": depth_range, "Average Accuracy": accuracies})
 df = df[["Max Depth", "Average Accuracy"]]
print(df.to_string(index=False))<choose_model_class>
def question_average_sum_by_user(df,question_df):
my_dict = {}
group = df.groupby(['user_id'])
 for user, val in group:
average_sum = 0.0
for row_index, row in val.iterrows() :
if(row['content_type_id'] == False):
question_id = row['content_id']
question_average = question_df.at[question_id,'question_average']
average_sum += question_average
my_dict[user] = [average_sum]
return pd.DataFrame.from_dict(my_dict,orient='index',columns=['question_average_sum'] )
y_train = train['Survived']
x_train = train.drop(['Survived'], axis=1 ).values
x_test = test.values
decision_tree = tree.DecisionTreeClassifier(max_depth = 3)
 decision_tree.fit(x_train, y_train)
 y_pred = decision_tree.predict(x_test)
 submission = pd.DataFrame({"PassengerId": PassengerId,"Survived": y_pred})
 submission.to_csv('submission.csv', index=False)
 print(submission.head(4))<save_to_csv>
def add_answers_to_prior_df(current_df,prior_df):
prior_df_ = prior_df.copy()
if(prior_df.shape[0] > 0):
val = eval(current_df.iloc[0]['prior_group_answers_correct'])
 if(len(val)== prior_df.shape[0]):
prior_df_['answered_correctly_response'] = val
return prior_df_
with open("tree1.dot", 'w')as f:
f = tree.export_graphviz(decision_tree,
out_file=f,
max_depth = 3,
impurity = True,
feature_names = list(train.drop(['Survived'], axis=1)) ,
class_names = ['Died', 'Survived'],
rounded = True,
filled= True)
 check_call(['dot','-Tpng','tree1.dot','-o','tree1.png'])
 img = Image.open("tree1.png")
 draw = ImageDraw.Draw(img)
 font = ImageFont.truetype('/usr/share/fonts/truetype/liberation/LiberationSerif-Bold.ttf', 26)
 draw.text(( 10, 0),
'"Title <= 1.5" corresponds to "Mr." title', 
(0,0,255),
font=font)
 img.save('sample-out.png')
 PImage("sample-out.png" )<compute_test_metric>
def build_question_df(prior_df,question_df):
if(prior_df.shape[0] == 0):
return question_df
question_sum_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['content_id'])['answered_correctly_response'].sum())\ 
.rename(columns={'answered_correctly_response':'question_sum'})
 question_count_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['content_id'] ).size() ,columns=['question_count'])
 question_df = question_df.join(question_sum_prior,rsuffix='_previous' ).join(question_count_prior,rsuffix='_previous')
 question_df['question_average'].fillna(0,inplace=True)
 question_df['question_count'].fillna(0,inplace=True)
 question_df['question_sum'].fillna(0,inplace=True)
 question_df['question_sum_previous'].fillna(0,inplace=True)
 question_df['question_count_previous'].fillna(0,inplace=True)
 question_df['question_sum'] = question_df['question_sum'] + question_df['question_sum_previous']
question_df['question_count'] = question_df['question_count'] + question_df['question_count_previous']
question_df['question_average'] = question_df['question_sum'] / question_df['question_count']
question_df.drop(['question_count_previous','question_sum_previous'],inplace=True,axis=1)
 return question_df
acc_decision_tree = round(decision_tree.score(x_train, y_train)* 100, 2)
 acc_decision_tree<set_options>
def build_user_df(prior_df,user_df,question_df):
if(prior_df.shape[0] == 0):
return user_df
user_sum_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['user_id'])['answered_correctly_response'].sum())\ 
.rename(columns={'answered_correctly_response':'user_sum'})
 user_count_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['user_id'] ).size() ,columns=['user_count'])
 user_df = user_df.join(user_sum_prior,how='outer',rsuffix='_previous' ).join(user_count_prior,rsuffix='_previous')
 user_df['performance'].fillna(0,inplace=True)
 user_df['user_average'].fillna(0,inplace=True)
 user_df['user_count'].fillna(0,inplace=True)
 user_df['user_sum'].fillna(0,inplace=True)
 user_df['user_count_previous'].fillna(0,inplace=True)
 user_df['user_sum_previous'].fillna(0,inplace=True)
 user_df = user_df.join(question_average_sum_by_user(prior_df,question_df))
user_df['question_average_sum'].fillna(0,inplace=True)
 user_df['user_mean_performance'] =(user_df['user_sum'] - user_df['performance'] * user_df['user_count'] + user_df['question_average_sum'])/(user_df['user_count'] + user_df['user_count_previous'])
 user_df['user_sum'] = user_df['user_sum'] + user_df['user_sum_previous']
user_df['user_count'] = user_df['user_count'] + user_df['user_count_previous']
user_df['user_average'] = user_df['user_sum'] / user_df['user_count']
user_df['performance'] = user_df['user_average'] - user_df['user_mean_performance']
user_df.drop(['user_sum_previous','user_count_previous','question_average_sum','user_mean_performance'],axis=1,inplace=True)
 return user_df
%matplotlib inline 
<load_from_csv>
prior_df = pd.DataFrame()
current_df = pd.DataFrame()
prior_df = add_answers_to_prior_df(current_df,prior_df)
 question_df = build_question_df(prior_df,question_df)
 user_df = build_user_df(prior_df,user_df,question_df )
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 combine = [train_df,test_df]
print(combine )<sort_values>
TARGET_COL = ['answered_correctly']
FEATURE_COLS = ['row_id', 'performance', 'question_average','content_id','content_type_id']
train_df[['Pclass','Survived']].groupby(['Pclass'],as_index=False ).mean().sort_values(by='Survived',ascending=False )<sort_values>
def data_transform(df, is_training = True, is_validation = True):
df = df.join(question_df['question_average'],on=['content_id'],rsuffix='_question_average')
 df = df.join(user_df[['performance','user_average', 'user_count']],on=['user_id'],rsuffix='_right')
 df['is_beginning'] = df['user_count'] < 20
df['attempt'] = df.groupby(['user_id', 'content_id'] ).content_id.transform('cumcount' ).astype(np.uint8)
 df = df.loc[df['content_type_id'] == False]
if is_training or is_validation:
df = df[FEATURE_COLS + TARGET_COL]
else:
df = df[FEATURE_COLS]
return df
train_df[['Sex','Survived']].groupby(['Sex'],as_index=False ).mean().sort_values(by='Survived',ascending=False )<sort_values>
%%time
train = data_transform(train )
train_df[['SibSp','Survived']].groupby(['SibSp'],as_index=False ).mean().sort_values(by='Survived',ascending=False )<sort_values>
test = data_transform(test,False )
train_df[['Parch','Survived']].groupby(['Parch'],as_index=False ).mean().sort_values(by='Survived',ascending=False )<feature_engineering>
X_train = train[FEATURE_COLS]
y_train = train[TARGET_COL]
X_test = test[FEATURE_COLS]
y_test = test[TARGET_COL]
for dataset in combine:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.', expand=False)
 pd.crosstab(train_df['Title'], train_df['Sex'] )<feature_engineering>
params = {
'objective': 'binary',
'seed': 42,
'metric': 'auc',
'learning_rate': 0.05,
'max_bin': 1000,
'num_leaves': 80,
'num_iterations' : 20
}
lgb_train = lgb.Dataset(X_train.iloc[:,1:],y_train)
 lgb_val = lgb.Dataset(X_test.iloc[:,1:],y_test )
for dataset in combine:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',\
	'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 train_df[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()<categorify>
del X_train,y_train
gc.collect()
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
for dataset in combine:
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0)
 train_df.head()<drop_column>
model = lgb.train(
 params,
lgb_train,
valid_sets=[lgb_train,lgb_val],
verbose_eval=1,
num_boost_round=100,
early_stopping_rounds=10
)
train_df = train_df.drop(['Name', 'PassengerId'], axis=1)
 test_df = test_df.drop(['Name'], axis=1)
 combine = [train_df, test_df]
train_df.shape, test_df.shape<data_type_conversions>
%%time
predictions = pd.DataFrame(model.predict(X_test.iloc[:,1:]),index=X_test.index )
for dataset in combine:
dataset['Sex'] = dataset['Sex'].map({'female': 1, 'male': 0} ).astype(int)
 train_df.head()<define_variables>
roc_auc_score(y_test,predictions[0] )
guess_ages = np.zeros(( 2,3))
guess_ages<find_best_params>
env = riiideducation.make_env()
iter_test = env.iter_test()
iter_nb = 0
for(current_df, sample_prediction_df)in iter_test:
print(current_df)
 if(iter_nb != 0):
prior_df = add_answers_to_prior_df(current_df,prior_df)
 question_df = build_question_df(prior_df,question_df)
 user_df = build_user_df(prior_df,user_df,question_df)
 prior_df = current_df.copy()
current_df = data_transform(current_df,False,False)
 current_df['answered_correctly'] = model.predict(current_df.iloc[:,1:])
 iter_nb = 1
env.predict(current_df.loc[:, ['row_id', 'answered_correctly']] )