Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<predict_on_test>
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1
print('The remaining 258 males are predicted to die')
 print('and the remaining 137 females are predicted to live' )<save_to_csv>
import numpy as np
import random
import pandas as pd
output = pd.DataFrame({'PassengerId': data[891:1309].PassengerId, 'Survived': data[891:1309].Predict.astype('int')})
 output.to_csv('WCG_gender.csv', index=False)
 print('WCG_gender submission was successfully saved!')
 print('Submission is loading...you scored 81,6%!' )<correct_missing_values>
_ = np.seterr(divide='ignore', invalid='ignore' )
def fix_fare(row):
if row.Fare == 0:
row.Fare = np.NaN
return row
print('The following {:d} passengers have a zero Fare:'.format(data[data.Fare==0].shape[0]))
cols = ['PassengerId', 'Survived', 'Pclass','Fare', 'Name']
data.loc[data.Fare==0, cols]<categorify>
data_types_dict = {
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id':'int8',
'task_container_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'
data['Ticket_freq'] = data.groupby('Ticket')['Ticket'].transform('count')
 data['Pfare'] = data['Fare'] / data['Ticket_freq']<split>
print('start read train data...')
 train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()
train_male = data[0:891].loc[(data.Sex=='male')&(data.WCSurvived.isnull())]
test_male = data[891:1309].loc[(data.Sex=='male')&(data.WCSurvived.isnull())]<prepare_x_and_y>
print('start handle lecture data...' )
cols = ['PassengerId', 'Name', 'Pfare', 'Pclass', 'Embarked']
y_m = train_male['Survived']
features = ['Pfare', 'Pclass', 'Embarked']
X_m = train_male[features]
numerical_cols = ['Pfare']
categorical_cols = ['Pclass', 'Embarked']
numerical_transformer = Pipeline(steps=[ 
('imputer', SimpleImputer()), 
('scaler', StandardScaler())
 ])
 categorical_transformer = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='most_frequent')) , 
('onehot', OneHotEncoder(handle_unknown='ignore'))
])
 preprocessor = ColumnTransformer(transformers=[ 
('num', numerical_transformer, numerical_cols), 
('cat', categorical_transformer, categorical_cols)
 ])
 precision_m = []
recall_m = []
for k in range(1,18):
pipeline1 = Pipeline(steps=[ 
('preprocessor', preprocessor), 
('model', KNeighborsClassifier(n_neighbors=k))
])
 precision_m.append(cross_val_score(pipeline1, X_m, y_m, cv=15, n_jobs=-1, scoring='precision' ).mean())
 recall_m.append(cross_val_score(pipeline1, X_m, y_m, cv=15, n_jobs=-1, scoring='recall' ).mean())
 k_range = range(1,18)
 plt.figure(figsize=(7,5))
plt.plot(k_range, precision_m, label='15-fold precision')
 plt.plot(k_range, recall_m, label='15-fold recall')
 plt.axhline(y=0.5, color='r')
 plt.xlabel('Value of k for KNN')
 plt.title('Precision and recall by number of neighbors', fontsize=14)
 plt.legend()
plt.show()<compute_train_metric>
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )
m1 = KNeighborsClassifier(n_neighbors=1)
 m2 = KNeighborsClassifier(n_neighbors=3)
 m3 = KNeighborsClassifier(n_neighbors=7)
 male_pipeline = Pipeline(steps=[ 
('preprocessor', preprocessor), 
('voting',VotingClassifier([ 
('m1', m1),('m2', m2),('m3', m3)]))
])
 print('15-fold precision of the ensemble: {:.3f}'.format(
 cross_val_score(male_pipeline, X_m, y_m, cv=15, n_jobs=-1, scoring='precision' ).mean()))
print('15-fold recall of the ensemble: {:.3f}'.format(
 cross_val_score(male_pipeline, X_m, y_m, cv=15, n_jobs=-1, scoring='recall' ).mean()))
print('15-fold accuracy of the ensemble: {:.3f}'.format(
 cross_val_score(male_pipeline, X_m, y_m, cv=15, n_jobs=-1 ).mean()))
male_pipeline.fit(X_m, y_m)
 learn_train_m = male_pipeline.predict(X_m)
 X_test_m = test_male[features]
predictions_m = male_pipeline.predict(X_test_m)
 print('
The following 9 adult males are predicted to live:')
 test_male.loc[(predictions_m==1), cols]<save_to_csv>
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]
data.loc[891:1308, 'Predict'][(data.Sex=='male')&(data.WCSurvived.isnull())] = predictions_m
output = pd.DataFrame({'PassengerId': data[891:1309].PassengerId, 'Survived': data[891:1309].Predict.astype('int')})
 output.to_csv('WCG_male.csv', index=False)
 print('WCG_male submission was successfully saved!')
 print('Submission is loading...you scored 82,3%!' )<split>
train_lectures = train_df[train_df.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )
train_female = data[0:891].loc[(data.Sex=='female')&(data.WCSurvived.isnull())]
test_female = data[891:1309].loc[(data.Sex=='female')&(data.WCSurvived.isnull())]<choose_model_class>
user_lecture_stats_part = train_lectures.groupby('user_id',as_index = False)[part_lectures_columns + types_of_lectures_columns].sum()
custom_precision = make_scorer(precision_score, pos_label=0, zero_division=0)
 custom_recall = make_scorer(recall_score, pos_label=0 )<compute_train_metric>
lecturedata_types_dict = {
'user_id': 'int32',
'part_1': 'int8',
'part_2': 'int8',
'part_3': 'int8',
'part_4': 'int8',
'part_5': 'int8',
'part_6': 'int8',
'part_7': 'int8',
'type_of_concept': 'int8',
'type_of_intention': 'int8',
'type_of_solving_question': 'int8',
'type_of_starter': 'int8'
}
user_lecture_stats_part = user_lecture_stats_part.astype(lecturedata_types_dict )
f1 = KNeighborsClassifier(n_neighbors=4)
 f2 = KNeighborsClassifier(n_neighbors=9)
 f3 = KNeighborsClassifier(n_neighbors=11)
 female_pipeline = Pipeline(steps=[ 
('preprocessor', preprocessor), 
('voting', VotingClassifier([ 
('f1', f1),('f2', f2),('f3', f3)]))
])
 print('9-fold precision of the ensemble: {:.3f}'.format(
 cross_val_score(female_pipeline, X_f, y_f, cv=9, scoring=custom_precision ).mean()))
print('9-fold recall of the ensemble: {:.3f}'.format(
 cross_val_score(female_pipeline, X_f, y_f, cv=9, scoring=custom_recall ).mean()))
print('9-fold accuracy of the ensemble: {:.3f}'.format(
 cross_val_score(female_pipeline, X_f, y_f, cv=9 ).mean()))
female_pipeline.fit(X_f, y_f)
 learn_train_f = female_pipeline.predict(X_f)
 X_test_f = test_female[features]
predictions_f = female_pipeline.predict(X_test_f)
 print('
The following 6 non-WCG females are predicted to die:')
 test_female.loc[(predictions_f==0), cols]<save_to_csv>
for column in user_lecture_stats_part.columns:
if(column !='user_id'):
user_lecture_stats_part[column] =(user_lecture_stats_part[column] > 0 ).astype('int8' )
data.loc[891:1308, 'Predict'][(data.Sex=='female')&(data.WCSurvived.isnull())] = predictions_f
output = pd.DataFrame({'PassengerId': data[891:1309].PassengerId, 'Survived': data[891:1309].Predict.astype('int')})
 output.to_csv('WCG_male_female.csv', index=False)
 print('WCG_male_female was successfully saved!')
 print('Submission is loading...you scored 82,8%!' )<load_from_csv>
del(train_lectures)
 gc.collect()
data_path = '/kaggle/input/titanic/'
train_df = pd.read_csv(data_path+'train.csv')
 test_df = pd.read_csv(data_path+'test.csv' )<concatenate>
user_lecture_agg = train_df.groupby('user_id')['content_type_id'].agg(['sum', 'count'])
 user_lecture_agg=user_lecture_agg.astype('int16' )
combined_df = pd.concat([train_df,test_df])
 train_df.name = 'Training Dataset'
test_df.name = 'Test Dataset'
combined_df.name = 'Combined Dataset'<count_missing_values>
cum = train_df.groupby('user_id')['content_type_id'].agg(['cumsum', 'cumcount'])
 cum['cumcount']=cum['cumcount']+1
train_df['user_interaction_count'] = cum['cumcount']
train_df['user_interaction_timestamp_mean'] = train_df['timestamp']/cum['cumcount']
train_df['user_lecture_sum'] = cum['cumsum']
train_df['user_lecture_lv'] = cum['cumsum'] / cum['cumcount']
train_df.user_lecture_lv=train_df.user_lecture_lv.astype('float16')
 train_df.user_lecture_sum=train_df.user_lecture_sum.astype('int16')
 train_df.user_interaction_count=train_df.user_interaction_count.astype('int16')
 train_df['user_interaction_timestamp_mean']=train_df['user_interaction_timestamp_mean']/(1000*3600)
 train_df.user_interaction_timestamp_mean=train_df.user_interaction_timestamp_mean.astype('float32')
def missing_columns(df):
for col in df.columns.tolist() :
print('{} column missing values: {}'.format(col, df[col].isnull().sum()))
print('
')
 for df in [train_df,test_df]:
print('{}'.format(df.name))
missing_columns(df )<set_options>
del cum
gc.collect()
%matplotlib inline 
<define_variables>
print('start handle train_df...' )
mask_m1 =(train_df.Sex == "male")&(train_df.Pclass == 1)
 mask_m2 =(train_df.Sex == "male")&(train_df.Pclass == 2)
 mask_m3 =(train_df.Sex == "male")&(train_df.Pclass == 3)
 mask_f1 =(train_df.Sex == "female")&(train_df.Pclass == 1)
 mask_f2 =(train_df.Sex == "female")&(train_df.Pclass == 2)
 mask_f3 =(train_df.Sex == "female")&(train_df.Pclass == 3)
 m_age_class1_male = combined_df.loc[(combined_df.Sex == "male")&(combined_df.Pclass == 1),'Age'].dropna().median()
m_age_class2_male = combined_df.loc[(combined_df.Sex == "male")&(combined_df.Pclass == 2),'Age'].dropna().median()
m_age_class3_male = combined_df.loc[(combined_df.Sex == "male")&(combined_df.Pclass == 3),'Age'].dropna().median()
m_age_class1_female = combined_df.loc[(combined_df.Sex == "female")&(combined_df.Pclass == 1),'Age'].dropna().median()
m_age_class2_female = combined_df.loc[(combined_df.Sex == "female")&(combined_df.Pclass == 2),'Age'].dropna().median()
m_age_class3_female = combined_df.loc[(combined_df.Sex == "female")&(combined_df.Pclass == 3),'Age'].dropna().median()
train_df.loc[mask_m1,'Age'] = train_df.loc[mask_m1,'Age'].fillna(m_age_class1_male)
 train_df.loc[mask_m2,'Age'] = train_df.loc[mask_m2,'Age'].fillna(m_age_class2_male)
 train_df.loc[mask_m3,'Age'] = train_df.loc[mask_m3,'Age'].fillna(m_age_class3_male)
 train_df.loc[mask_f1,'Age'] = train_df.loc[mask_f1,'Age'].fillna(m_age_class1_female)
 train_df.loc[mask_f2,'Age'] = train_df.loc[mask_f2,'Age'].fillna(m_age_class2_female)
 train_df.loc[mask_f3,'Age'] = train_df.loc[mask_f3,'Age'].fillna(m_age_class3_female)
<feature_engineering>
train_df['prior_question_had_explanation'].fillna(False, inplace=True)
 train_df = train_df.astype(data_types_dict)
 train_df = train_df[train_df[target] != -1].reset_index(drop=True )
mask_m1 =(test_df.Sex == "male")&(test_df.Pclass == 1)
 mask_m2 =(test_df.Sex == "male")&(test_df.Pclass == 2)
 mask_m3 =(test_df.Sex == "male")&(test_df.Pclass == 3)
 mask_f1 =(test_df.Sex == "female")&(test_df.Pclass == 1)
 mask_f2 =(test_df.Sex == "female")&(test_df.Pclass == 2)
 mask_f3 =(test_df.Sex == "female")&(test_df.Pclass == 3)
 test_df.loc[mask_m1,'Age'] = test_df.loc[mask_m1,'Age'].fillna(m_age_class1_male)
 test_df.loc[mask_m2,'Age'] = test_df.loc[mask_m2,'Age'].fillna(m_age_class2_male)
 test_df.loc[mask_m3,'Age'] = test_df.loc[mask_m3,'Age'].fillna(m_age_class3_male)
 test_df.loc[mask_f1,'Age'] = test_df.loc[mask_f1,'Age'].fillna(m_age_class1_female)
 test_df.loc[mask_f2,'Age'] = test_df.loc[mask_f2,'Age'].fillna(m_age_class2_female)
 test_df.loc[mask_f3,'Age'] = test_df.loc[mask_f3,'Age'].fillna(m_age_class3_female )<filter>
content_explation_agg=train_df[["content_id","prior_question_had_explanation",target]].groupby(["content_id","prior_question_had_explanation"])[target].agg(['mean'] )
train_df[train_df.Embarked.isna() ]<data_type_conversions>
content_explation_agg=content_explation_agg.unstack()
content_explation_agg=content_explation_agg.reset_index()
content_explation_agg.columns = ['content_id', 'content_explation_false_mean','content_explation_true_mean']
train_df['Embarked'] = train_df['Embarked'].fillna('C' )<count_missing_values>
content_explation_agg.content_id=content_explation_agg.content_id.astype('int16')
 content_explation_agg.content_explation_false_mean=content_explation_agg.content_explation_false_mean.astype('float16')
 content_explation_agg.content_explation_true_mean=content_explation_agg.content_explation_true_mean.astype('float16' )
median_class3_fare = combined_df.loc[(combined_df.Pclass == 3),'Fare'].dropna().median()
test_df.Fare.fillna(median_class3_fare,inplace=True)
<feature_engineering>
print('start handle attempt_no...' )
train_df['SibSp_cat'] =(train_df['SibSp'] > 0)* 1
train_df['Parch_cat'] =(train_df['Parch'] > 0)* 1
train_df['Gender'] =(train_df['Sex'] == 'female')* 1
train_df['Class 1'] =(train_df['Pclass'] == 1)* 1
train_df['Class 2'] =(train_df['Pclass'] == 2)* 1
train_df['Class 3'] =(train_df['Pclass'] == 3)* 1
train_df['IsAlone'] =(( train_df['SibSp'] == 0)&(train_df['Parch'] == 0)) * 1
train_df['High_Fare'] =(train_df['Fare'] >= 200.0)* 1
train_df['Family_Members'] = train_df.SibSp + train_df.Parch<feature_engineering>
train_df["attempt_no"] = 1
train_df.attempt_no=train_df.attempt_no.astype('int8')
 attempt_no_agg=train_df.groupby(["user_id","content_id"])["attempt_no"].agg(['sum'] ).astype('int8')
 train_df["attempt_no"] = train_df[["user_id","content_id",'attempt_no']].groupby(["user_id","content_id"])["attempt_no"].cumsum()
test_df['SibSp_cat'] =(test_df['SibSp'] > 0)* 1
test_df['Parch_cat'] =(test_df['Parch'] > 0)* 1
test_df['Gender'] =(test_df['Sex'] == 'female')* 1
test_df['Class 1'] =(test_df['Pclass'] == 1)* 1
test_df['Class 2'] =(test_df['Pclass'] == 2)* 1
test_df['Class 3'] =(test_df['Pclass'] == 3)* 1
test_df['IsAlone'] =(( test_df['SibSp'] == 0)&(test_df['Parch'] == 0)) * 1
test_df['High_Fare'] =(test_df['Fare'] >= 200.0)* 1
test_df['Family_Members'] = test_df.SibSp + test_df.Parch<prepare_x_and_y>
print('start handle timestamp...')
 prior_question_elapsed_time_mean=train_df['prior_question_elapsed_time'].mean()
train_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True )
y_train = train_df['Survived']
features = ['Gender','Age','Class 1','Class 2','Class 3','Family_Members','Embarked','Fare']
X_train = train_df[features]
X_test = test_df[features]<categorify>
max_timestamp_u = train_df[['user_id','timestamp']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u.columns = ['user_id', 'max_time_stamp']
max_timestamp_u.user_id=max_timestamp_u.user_id.astype('int32' )
X_train_1hot = pd.get_dummies(X_train)
 X_train_1hot<categorify>
train_df['lagtime'] = train_df.groupby('user_id')['timestamp'].shift()
max_timestamp_u2 = train_df[['user_id','lagtime']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u2.columns = ['user_id', 'max_time_stamp2']
max_timestamp_u2.user_id=max_timestamp_u2.user_id.astype('int32' )
X_test_1hot = pd.get_dummies(X_test)
 X_test_1hot<train_model>
train_df['lagtime']=train_df['timestamp']-train_df['lagtime']
lagtime_mean=train_df['lagtime'].mean()
train_df['lagtime'].fillna(lagtime_mean, inplace=True )
base_model = RandomForestClassifier(random_state=42,n_estimators=100,max_depth=5)
 fit = base_model.fit(X_train_1hot,y_train )<train_model>
train_df['lagtime']=train_df['lagtime']/(1000*3600)
 train_df.lagtime=train_df.lagtime.astype('float32' )
fit.score(X_train_1hot,y_train )<define_search_space>
n_estimators = [int(x)for x in np.linspace(start = 100, stop = 1000, num = 10)]
max_features = ['auto', 'sqrt']
max_depth = [5,6,7,8,9,10]
max_depth.append(None)
 min_samples_split = [2, 5, 10]
min_samples_leaf = [1, 2, 4]
bootstrap = [True, False]
random_grid = {'n_estimators': n_estimators,
'max_features': max_features,
'max_depth': max_depth,
'min_samples_split': min_samples_split,
'min_samples_leaf': min_samples_leaf,
'bootstrap': bootstrap}
pp = pprint.PrettyPrinter(indent=4)
 pp.pprint(random_grid )<train_on_grid>
train_df['lagtime2'] = train_df.groupby('user_id')['timestamp'].shift(2)
 max_timestamp_u3 = train_df[['user_id','lagtime2']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u3.columns = ['user_id', 'max_time_stamp3']
max_timestamp_u3.user_id=max_timestamp_u3.user_id.astype('int32')
 train_df['lagtime2']=train_df['timestamp']-train_df['lagtime2']
lagtime_mean2=train_df['lagtime2'].mean()
train_df['lagtime2'].fillna(lagtime_mean2, inplace=True)
rf = RandomForestClassifier()
rf_random = RandomizedSearchCV(estimator = rf, param_distributions = random_grid, n_iter = 200, cv = 4, verbose=2, random_state=42, n_jobs = -1)
 rf_random.fit(X_train_1hot, y_train)
<find_best_params>
train_df['lagtime2']=train_df['lagtime2']/(1000*3600)
 train_df.lagtime2=train_df.lagtime2.astype('float32' )
rf_random.best_params_<train_model>
train_df['lagtime3'] = train_df.groupby('user_id')['timestamp'].shift(3)
 train_df['lagtime3']=train_df['timestamp']-train_df['lagtime3']
lagtime_mean3=train_df['lagtime3'].mean()
train_df['lagtime3'].fillna(lagtime_mean3, inplace=True)
 train_df['lagtime3']=train_df['lagtime3']/(1000*3600)
 train_df.lagtime3=train_df.lagtime3.astype('float32' )
rf_best = rf_random.best_estimator_
best_fit = rf_best.fit(X_train_1hot, y_train)
 best_fit.score(X_train_1hot,y_train )<save_to_csv>
predictions = base_model.predict(X_test_1hot)
 output = pd.DataFrame({'PassengerId': test_df.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )<load_from_csv>
train_df['timestamp']=train_df['timestamp']/(1000*3600)
 train_df.timestamp=train_df.timestamp.astype('float16' )
sub1=pd.read_csv('.. /input/titanic-leaked/titanic.csv' )<save_to_csv>
train_df['delta_prior_question_elapsed_time'] = train_df.groupby('user_id')['prior_question_elapsed_time'].shift()
train_df['delta_prior_question_elapsed_time']=train_df['prior_question_elapsed_time']-train_df['delta_prior_question_elapsed_time']
sub1.to_csv('submission1.csv', index=False )<set_options>
delta_prior_question_elapsed_time_mean=train_df['delta_prior_question_elapsed_time'].mean()
train_df['delta_prior_question_elapsed_time'].fillna(delta_prior_question_elapsed_time_mean, inplace=True)
 train_df.delta_prior_question_elapsed_time=train_df.delta_prior_question_elapsed_time.astype('int32' )
%matplotlib inline 
<load_from_csv>
train_df['lag'] = train_df.groupby('user_id')[target].shift()
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 user_agg = train_df.groupby('user_id')['lag'].agg(['sum', 'count'] ).astype('int16')
 cum['cumsum'].fillna(0, inplace=True)
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df['user_correct_count'] = cum['cumsum']
train_df['user_uncorrect_count'] = cum['cumcount']-cum['cumsum']
train_df.drop(columns=['lag'], inplace=True)
 train_df['user_correctness'].fillna(0.67, inplace=True)
 train_df.user_correctness=train_df.user_correctness.astype('float16')
 train_df.user_correct_count=train_df.user_correct_count.astype('int16')
 train_df.user_uncorrect_count=train_df.user_uncorrect_count.astype('int16')
train_df = pd.read_csv('.. /input/titanic/train.csv')
 test_df = pd.read_csv('.. /input/titanic/test.csv')
 combine = [train_df, test_df]<sort_values>
del cum
gc.collect()
train_df[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<sort_values>
train_df[["Sex", "Survived"]].groupby(['Sex'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<sort_values>
train_df.prior_question_had_explanation=train_df.prior_question_had_explanation.astype('int8')
 explanation_agg = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['sum', 'count'])
 explanation_agg=explanation_agg.astype('int16')
train_df[["SibSp", "Survived"]].groupby(['SibSp'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<sort_values>
cum = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['cumsum', 'cumcount'])
 cum['cumcount']=cum['cumcount']+1
train_df['explanation_mean'] = cum['cumsum'] / cum['cumcount']
train_df['explanation_true_count'] = cum['cumsum']
train_df['explanation_false_count'] = cum['cumcount']-cum['cumsum']
train_df.explanation_mean=train_df.explanation_mean.astype('float16')
 train_df.explanation_true_count=train_df.explanation_true_count.astype('int16')
 train_df.explanation_false_count=train_df.explanation_false_count.astype('int16' )
train_df[["Parch", "Survived"]].groupby(['Parch'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<feature_engineering>
del cum
gc.collect()
for dataset in combine:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.', expand=False)
 pd.crosstab(train_df['Title'], train_df['Sex'] )<feature_engineering>
content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count','var'])
 task_container_agg = train_df.groupby('task_container_id')[target].agg(['sum', 'count','var'])
 content_agg=content_agg.astype('float32')
 task_container_agg=task_container_agg.astype('float32' )
for dataset in combine:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',\
	'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 train_df[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()<categorify>
train_df['task_container_uncor_count'] = train_df['task_container_id'].map(task_container_agg['count']-task_container_agg['sum'] ).astype('int32')
 train_df['task_container_cor_count'] = train_df['task_container_id'].map(task_container_agg['sum'] ).astype('int32')
 train_df['task_container_std'] = train_df['task_container_id'].map(task_container_agg['var'] ).astype('float16')
 train_df['task_container_correctness'] = train_df['task_container_id'].map(task_container_agg['sum'] / task_container_agg['count'])
 train_df.task_container_correctness=train_df.task_container_correctness.astype('float16' )
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
for dataset in combine:
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0)
 train_df.head()<drop_column>
content_elapsed_time_agg=train_df.groupby('content_id')['prior_question_elapsed_time'].agg(['mean'])
 content_had_explanation_agg=train_df.groupby('content_id')['prior_question_had_explanation'].agg(['mean'] )
train_df = train_df.drop(['Name', 'PassengerId'], axis=1)
 test_df = test_df.drop(['Name'], axis=1)
 combine = [train_df, test_df]
train_df.shape, test_df.shape<data_type_conversions>
print('start questions data...' )
for dataset in combine:
dataset['Sex'] = dataset['Sex'].map({'female': 1, 'male': 0} ).astype(int)
 train_df.head()<define_variables>
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 1,3,4],
dtype={'question_id': 'int16','bundle_id': 'int16', 'part': 'int8','tags': 'str'}
)
guess_ages = np.zeros(( 2,3))
guess_ages<find_best_params>
bundle_agg = questions_df.groupby('bundle_id')['question_id'].agg(['count'] )
for dataset in combine:
for i in range(0, 2):
for j in range(0, 3):
guess_df = dataset[(dataset['Sex'] == i)& \ 
(dataset['Pclass'] == j+1)]['Age'].dropna()
age_guess = guess_df.median()
guess_ages[i,j] = int(age_guess/0.5 + 0.5)* 0.5
for i in range(0, 2):
for j in range(0, 3):
dataset.loc[(dataset.Age.isnull())&(dataset.Sex == i)&(dataset.Pclass == j+1),\
'Age'] = guess_ages[i,j]
dataset['Age'] = dataset['Age'].astype(int)
 train_df.head()<sort_values>
questions_df['content_sub_bundle'] = questions_df['bundle_id'].map(bundle_agg['count'] ).astype('int8' )
train_df['AgeBand'] = pd.cut(train_df['Age'], 5)
 train_df[['AgeBand', 'Survived']].groupby(['AgeBand'], as_index=False ).mean().sort_values(by='AgeBand', ascending=True )<feature_engineering>
questions_df['tags'].fillna('188', inplace=True )
for dataset in combine:
dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age']
train_df.head()<drop_column>
def gettags(tags,num):
tags_splits=tags.split(" ")
 result=''
for t in tags_splits:
x=int(t)
 if(x<32*(num+1)and x>=32*num):
result=result+' '+t
return result
train_df = train_df.drop(['AgeBand'], axis=1)
 combine = [train_df, test_df]
train_df.head()<sort_values>
for num in range(0,6):
questions_df["tags"+str(num)] = questions_df["tags"].apply(lambda row: gettags(row,num))
le = LabelEncoder()
le.fit(np.unique(questions_df['tags'+str(num)].values))
questions_df['tags'+str(num)]=questions_df[['tags'+str(num)]].apply(le.transform )
for dataset in combine:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
train_df[['FamilySize', 'Survived']].groupby(['FamilySize'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<feature_engineering>
questions_df_dict = {
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
}
questions_df = questions_df.astype(questions_df_dict )
for dataset in combine:
dataset['IsAlone'] = 0
dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1
train_df[['IsAlone', 'Survived']].groupby(['IsAlone'], as_index=False ).mean()<drop_column>
questions_df.drop(columns=['tags'], inplace=True )
train_df = train_df.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
 test_df = test_df.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
 combine = [train_df, test_df]
train_df.head()<feature_engineering>
questions_df['part_bundle_id']=questions_df['part']*100000+questions_df['bundle_id']
questions_df.part_bundle_id=questions_df.part_bundle_id.astype('int32')
for dataset in combine:
dataset['Age*Class'] = dataset.Age * dataset.Pclass
train_df.loc[:, ['Age*Class', 'Age', 'Pclass']].head(10 )<correct_missing_values>
questions_cmnts = pd.read_csv(
 '.. /input/2020-r3id-clustering-question-tags/question_cmnts.csv',
usecols=[1,2],
dtype={'question_id': 'int16','community': 'int8'}
)
freq_port = train_df.Embarked.dropna().mode() [0]
freq_port<sort_values>
questions_df = pd.merge(questions_df, questions_cmnts, on='question_id', how='left',right_index=True )
for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].fillna(freq_port)
 train_df[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<data_type_conversions>
questions_df.rename(columns={'question_id':'content_id'}, inplace=True )
for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 train_df.head()<data_type_conversions>
questions_df = pd.merge(questions_df, content_explation_agg, on='content_id', how='left',right_index=True)
test_df['Fare'].fillna(test_df['Fare'].dropna().median() , inplace=True)
 test_df.head()<sort_values>
del content_explation_agg
train_df['FareBand'] = pd.qcut(train_df['Fare'], 4)
 train_df[['FareBand', 'Survived']].groupby(['FareBand'], as_index=False ).mean().sort_values(by='FareBand', ascending=True )<data_type_conversions>
questions_df['content_correctness'] = questions_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 questions_df.content_correctness=questions_df.content_correctness.astype('float16')
 questions_df['content_correctness_std'] = questions_df['content_id'].map(content_agg['var'])
 questions_df.content_correctness_std=questions_df.content_correctness_std.astype('float16')
 questions_df['content_uncorrect_count'] = questions_df['content_id'].map(content_agg['count']-content_agg['sum'] ).astype('int32')
 questions_df['content_correct_count'] = questions_df['content_id'].map(content_agg['sum'] ).astype('int32' )
for dataset in combine:
dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] = 3
dataset['Fare'] = dataset['Fare'].astype(int)
 train_df = train_df.drop(['FareBand'], axis=1)
 combine = [train_df, test_df]
train_df.head(10 )<prepare_x_and_y>
questions_df['content_elapsed_time_mean'] = questions_df['content_id'].map(content_elapsed_time_agg['mean'])
 questions_df.content_elapsed_time_mean=questions_df.content_elapsed_time_mean.astype('float16')
 questions_df['content_had_explanation_mean'] = questions_df['content_id'].map(content_had_explanation_agg['mean'])
 questions_df.content_had_explanation_mean=questions_df.content_had_explanation_mean.astype('float16' )
X_train = train_df.drop("Survived", axis=1)
 Y_train = train_df["Survived"]
X_test = test_df.drop("PassengerId", axis=1 ).copy()
X_train.shape, Y_train.shape, X_test.shape<choose_model_class>
del content_elapsed_time_agg
del content_had_explanation_agg
gc.collect()
logreg = LogisticRegression()
logreg.fit(X_train, Y_train)
 Y_pred = logreg.predict(X_test)
 acc_log = round(logreg.score(X_train, Y_train)* 100, 2)
 acc_log<compute_train_metric>
part_agg = questions_df.groupby('part')['content_correctness'].agg(['mean', 'var'])
 questions_df['part_correctness_mean'] = questions_df['part'].map(part_agg['mean'])
 questions_df['part_correctness_std'] = questions_df['part'].map(part_agg['var'])
 questions_df.part_correctness_mean=questions_df.part_correctness_mean.astype('float16')
 questions_df.part_correctness_std=questions_df.part_correctness_std.astype('float16' )
svc = SVC()
svc.fit(X_train, Y_train)
 Y_pred = svc.predict(X_test)
 acc_svc = round(svc.score(X_train, Y_train)* 100, 2)
 acc_svc<predict_on_test>
part_agg = questions_df.groupby('part')['content_uncorrect_count'].agg(['sum'])
 questions_df['part_uncor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32')
 part_agg = questions_df.groupby('part')['content_correct_count'].agg(['sum'])
 questions_df['part_cor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32' )
knn = KNeighborsClassifier(n_neighbors = 3)
 knn.fit(X_train, Y_train)
 Y_pred = knn.predict(X_test)
 acc_knn = round(knn.score(X_train, Y_train)* 100, 2)
 acc_knn<predict_on_test>
bundle_agg = questions_df.groupby('bundle_id')['content_correctness'].agg(['mean'])
 questions_df['bundle_correctness_mean'] = questions_df['bundle_id'].map(bundle_agg['mean'])
 questions_df.bundle_correctness_mean=questions_df.bundle_correctness_mean.astype('float16')
gaussian = GaussianNB()
gaussian.fit(X_train, Y_train)
 Y_pred = gaussian.predict(X_test)
 acc_gaussian = round(gaussian.score(X_train, Y_train)* 100, 2)
 acc_gaussian<compute_train_metric>
perceptron = Perceptron()
perceptron.fit(X_train, Y_train)
 Y_pred = perceptron.predict(X_test)
 acc_perceptron = round(perceptron.score(X_train, Y_train)* 100, 2)
 acc_perceptron<compute_train_metric>
del content_agg
del bundle_agg
del part_agg
gc.collect()
linear_svc = LinearSVC()
linear_svc.fit(X_train, Y_train)
 Y_pred = linear_svc.predict(X_test)
 acc_linear_svc = round(linear_svc.score(X_train, Y_train)* 100, 2)
 acc_linear_svc<compute_train_metric>
features_dict = {
'timestamp':'float16',
'user_interaction_count':'int16',
'user_interaction_timestamp_mean':'float32',
'lagtime':'float32',
'lagtime2':'float32',
'lagtime3':'float32',
'content_id':'int16',
'task_container_id':'int16',
'user_lecture_sum':'int16',
'user_lecture_lv':'float16',
'prior_question_elapsed_time':'float32',
'delta_prior_question_elapsed_time':'int32',
'user_correctness':'float16',
'user_uncorrect_count':'int16',
'user_correct_count':'int16',
'content_correctness_std':'float16',
'content_correct_count':'int32',
'content_uncorrect_count':'int32',
'content_elapsed_time_mean':'float16',
'content_had_explanation_mean':'float16',
'content_explation_false_mean':'float16',
'content_explation_true_mean':'float16',
'task_container_correctness':'float16',
'task_container_std':'float16',
'task_container_cor_count':'int32',
'task_container_uncor_count':'int32',
'attempt_no':'int8',
'part':'int8',
'part_correctness_mean':'float16',
'part_correctness_std':'float16',
'part_uncor_count':'int32',
'part_cor_count':'int32',
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
'part_bundle_id':'int32',
'content_sub_bundle':'int8',
'prior_question_had_explanation':'int8',
'explanation_mean':'float16',
'explanation_false_count':'int16',
'explanation_true_count':'int16',
}
categorical_columns= [
'content_id',
'task_container_id',
'part',
'tags0',
'tags1',
'tags2',
'tags3',
'tags4',
'tags5',
'part_bundle_id',
'content_sub_bundle',
'prior_question_had_explanation',
]
features=list(features_dict.keys())
sgd = SGDClassifier()
sgd.fit(X_train, Y_train)
 Y_pred = sgd.predict(X_test)
 acc_sgd = round(sgd.score(X_train, Y_train)* 100, 2)
 acc_sgd<choose_model_class>
flag_lgbm=True
clfs = list()
params = {
'num_leaves': 300,
'max_bin':450,
'feature_fraction': 0.52,
'bagging_fraction': 0.52,
'objective': 'binary',
'learning_rate': 0.05,
"boosting_type": "gbdt",
"metric": 'auc',
}
trains=list()
valids=list()
num=1
for i in range(0,num):
train_df_clf=train_df[1200*10000:2*1200*10000]
print('sample end')
 del train_df
users=train_df_clf['user_id'].drop_duplicates()
users=users.sample(frac=0.08)
 users_df=pd.DataFrame()
users_df['user_id']=users.values
valid_df_newuser = pd.merge(train_df_clf, users_df, on=['user_id'], how='inner',right_index=True)
 del users_df
del users
gc.collect()
train_df_clf.drop(valid_df_newuser.index, inplace=True)
 print('pd.merge(train_df_clf, questions_df)')
 train_df_clf = pd.merge(train_df_clf, questions_df, on='content_id', how='left',right_index=True)
 valid_df_newuser = pd.merge(valid_df_newuser, questions_df, on='content_id', how='left',right_index=True)
 print('valid_df')
 valid_df=train_df_clf.sample(frac=0.1)
 train_df_clf.drop(valid_df.index, inplace=True)
 valid_df = valid_df.append(valid_df_newuser)
 del valid_df_newuser
gc.collect()
trains.append(train_df_clf)
 valids.append(valid_df)
 print('train_df_clf length：',len(train_df_clf))
print('valid_df length：',len(valid_df))
decision_tree = DecisionTreeClassifier()
decision_tree.fit(X_train, Y_train)
 Y_pred = decision_tree.predict(X_test)
 acc_decision_tree = round(decision_tree.score(X_train, Y_train)* 100, 2)
 acc_decision_tree<train_model>
del train_df_clf
del valid_df
gc.collect()
random_forest = RandomForestClassifier(n_estimators=100)
 random_forest.fit(X_train, Y_train)
 Y_pred = random_forest.predict(X_test)
 random_forest.score(X_train, Y_train)
 acc_random_forest = round(random_forest.score(X_train, Y_train)* 100, 2)
 acc_random_forest<create_dataframe>
for i in range(0,num):
X_train_np = trains[i][features].values.astype(np.float32)
 X_valid_np = valids[i][features].values.astype(np.float32)
 tr_data = lgb.Dataset(X_train_np, label=trains[i][target], feature_name=list(features))
va_data = lgb.Dataset(X_valid_np, label=valids[i][target], feature_name=list(features))
del trains
del valids
del X_train_np
del X_valid_np
gc.collect()
model = lgb.train(
 params,
tr_data,
num_boost_round=5000,
valid_sets=[tr_data, va_data],
early_stopping_rounds=50,
feature_name=features,
categorical_feature=categorical_columns,
verbose_eval=50 
)
 clfs.append(model)
 fig,ax = plt.subplots(figsize=(15,15))
lgb.plot_importance(model, ax=ax,importance_type='gain',max_num_features=50)
 plt.show()
del tr_data
del va_data
gc.collect()
models = pd.DataFrame({
'Model': ['Support Vector Machines', 'KNN', 'Logistic Regression',
'Random Forest', 'Naive Bayes', 'Perceptron',
'Stochastic Gradient Decent', 'Linear SVC',
'Decision Tree'],
'Score': [acc_svc, acc_knn, acc_log,
acc_random_forest, acc_gaussian, acc_perceptron,
acc_sgd, acc_linear_svc, acc_decision_tree]})
 sorted_model=models.sort_values(by='Score', ascending=False)
 sorted_model<save_to_csv>
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int))
submission = pd.DataFrame({
"PassengerId": test_df["PassengerId"],
"Survived": Y_pred
})
 submission.to_csv('submission2.csv', index=False )<import_modules>
del user_agg
gc.collect()
task_container_sum_dict = task_container_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
task_container_count_dict = task_container_agg['count'].astype('int32' ).to_dict(defaultdict(int))
task_container_std_dict = task_container_agg['var'].astype('float16' ).to_dict(defaultdict(int))
explanation_sum_dict = explanation_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
explanation_count_dict = explanation_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del task_container_agg
del explanation_agg
gc.collect()
import plotly.graph_objects as go
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tqdm import tqdm
import time
from datetime import datetime
from pathlib import Path
from sklearn import preprocessing
import keras.backend as K
from keras.models import Sequential
from keras.layers import Dense, LSTM, RNN, Dropout
from keras.callbacks import EarlyStopping, ModelCheckpoint
from keras import optimizers
from sklearn.preprocessing import StandardScaler, MinMaxScaler, LabelEncoder
from sklearn.model_selection import train_test_split<load_from_csv>
user_lecture_sum_dict = user_lecture_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_lecture_count_dict = user_lecture_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del user_lecture_agg
gc.collect()
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/train.csv', parse_dates=['Date'])
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/test.csv',parse_dates=['Date'])
 submission = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/submission.csv')
 train.tail()<define_variables>
max_timestamp_u_dict=max_timestamp_u.set_index('user_id' ).to_dict()
max_timestamp_u_dict2=max_timestamp_u2.set_index('user_id' ).to_dict()
max_timestamp_u_dict3=max_timestamp_u3.set_index('user_id' ).to_dict()
user_prior_question_elapsed_time_dict=user_prior_question_elapsed_time.set_index('user_id' ).to_dict()
del max_timestamp_u
del max_timestamp_u2
del max_timestamp_u3
del user_prior_question_elapsed_time
gc.collect()
mask = train['Date'].max()
world_cum_confirmed = sum(train[train['Date'] == mask].ConfirmedCases)
 world_cum_fatal = sum(train[train['Date'] == mask].Fatalities )<train_model>
attempt_no_sum_dict = attempt_no_agg['sum'].to_dict(defaultdict(int))
del attempt_no_agg
gc.collect()
print('Number of Countires are: ', len(train['Country_Region'].unique()))
print('Training dataset ends at: ', mask)
 print('Number of cumulative confirmed cases worldwide are: ', world_cum_confirmed)
 print('Number of cumulative fatal cases worldwide are: ', world_cum_fatal )<sort_values>
def get_max_attempt(user_id,content_id):
k =(user_id,content_id)
 if k in attempt_no_sum_dict.keys() :
attempt_no_sum_dict[k]+=1
return attempt_no_sum_dict[k]
attempt_no_sum_dict[k] = 1
return attempt_no_sum_dict[k]
cum_per_country = train[train['Date'] == mask].groupby(['Date','Country_Region'] ).sum().sort_values(['ConfirmedCases'], ascending=False)
 cum_per_country[:10]<groupby>
date = train['Date'].unique()
cc_us = train[train['Country_Region'] == 'US'].groupby(['Date'] ).sum().ConfirmedCases
ft_us = train[train['Country_Region'] == 'US'].groupby(['Date'] ).sum().Fatalities
cc_ity = train[train['Country_Region'] == 'Italy'].groupby(['Date'] ).sum().ConfirmedCases
ft_ity = train[train['Country_Region'] == 'Italy'].groupby(['Date'] ).sum().Fatalities
cc_spn = train[train['Country_Region'] == 'Spain'].groupby(['Date'] ).sum().ConfirmedCases
ft_spn = train[train['Country_Region'] == 'Spain'].groupby(['Date'] ).sum().Fatalities
cc_gmn = train[train['Country_Region'] == 'Germany'].groupby(['Date'] ).sum().ConfirmedCases
ft_gmn = train[train['Country_Region'] == 'Germany'].groupby(['Date'] ).sum().Fatalities
cc_frc = train[train['Country_Region'] == 'France'].groupby(['Date'] ).sum().ConfirmedCases
ft_frc = train[train['Country_Region'] == 'France'].groupby(['Date'] ).sum().Fatalities
fig = go.Figure()
fig.add_trace(go.Scatter(x=date, y=cc_us, name='US'))
fig.add_trace(go.Scatter(x=date, y=cc_ity, name='Italy'))
fig.add_trace(go.Scatter(x=date, y=cc_spn, name='Spain'))
fig.add_trace(go.Scatter(x=date, y=cc_gmn, name='Germany'))
fig.add_trace(go.Scatter(x=date, y=cc_frc, name='France'))
fig.update_layout(title="Plot of Cumulative Cases for Top 5 countires(except China)",
xaxis_title="Date",
yaxis_title="Cases")
 fig.update_xaxes(nticks=30)
 fig.show()<data_type_conversions>
iter_test = env.iter_test()
prior_test_df = None
train.columns = train.columns.str.lower()
test.columns = test.columns.str.lower()<drop_column>
N=[0.4,0.6]
<feature_engineering><EOS> 
%%time
for(test_df, sample_prediction_df)in iter_test:
if prior_test_df is not None:
prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)
 prior_test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 prior_test_df.prior_question_had_explanation=prior_test_df.prior_question_had_explanation.astype('int8')
 user_ids = prior_test_df['user_id'].values
targets = prior_test_df[target].values
for user_id, answered_correctly in zip(user_ids,targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
prior_test_df = test_df.copy()
question_len=len(test_df[test_df['content_type_id'] == 0])
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df.prior_question_had_explanation=test_df.prior_question_had_explanation.astype('int8')
 test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)
 user_lecture_sum = np.zeros(question_len, dtype=np.int16)
 user_lecture_count = np.zeros(question_len, dtype=np.int16)
 user_sum = np.zeros(question_len, dtype=np.int16)
 user_count = np.zeros(question_len, dtype=np.int16)
 task_container_sum = np.zeros(question_len, dtype=np.int32)
 task_container_count = np.zeros(question_len, dtype=np.int32)
 task_container_std = np.zeros(question_len, dtype=np.float16)
 explanation_sum = np.zeros(question_len, dtype=np.int32)
 explanation_count = np.zeros(question_len, dtype=np.int32)
 delta_prior_question_elapsed_time = np.zeros(question_len, dtype=np.int32)
 attempt_no_count = np.zeros(question_len, dtype=np.int16)
 lagtime = np.zeros(question_len, dtype=np.float32)
 lagtime2 = np.zeros(question_len, dtype=np.float32)
 lagtime3 = np.zeros(question_len, dtype=np.float32)
 i=0
for j,(user_id,prior_question_had_explanation,content_type_id,prior_question_elapsed_time,timestamp, content_id,task_container_id)in enumerate(zip(test_df['user_id'].values,test_df['prior_question_had_explanation'].values,test_df['content_type_id'].values,test_df['prior_question_elapsed_time'].values,test_df['timestamp'].values, test_df['content_id'].values, test_df['task_container_id'].values)) :
user_lecture_sum_dict[user_id] += content_type_id
user_lecture_count_dict[user_id] += 1
if(content_type_id==1):
x=1
if(content_type_id==0):
user_lecture_sum[i] = user_lecture_sum_dict[user_id]
user_lecture_count[i] = user_lecture_count_dict[user_id]
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
task_container_sum[i] = task_container_sum_dict[task_container_id]
task_container_count[i] = task_container_count_dict[task_container_id]
task_container_std[i]=task_container_std_dict[task_container_id]
explanation_sum_dict[user_id] += prior_question_had_explanation
explanation_count_dict[user_id] += 1
explanation_sum[i] = explanation_sum_dict[user_id]
explanation_count[i] = explanation_count_dict[user_id]
if user_id in max_timestamp_u_dict['max_time_stamp'].keys() :
lagtime[i]=timestamp-max_timestamp_u_dict['max_time_stamp'][user_id]
if(max_timestamp_u_dict2['max_time_stamp2'][user_id]==lagtime_mean2):
lagtime2[i]=lagtime_mean2
lagtime3[i]=lagtime_mean3
else:
lagtime2[i]=timestamp-max_timestamp_u_dict2['max_time_stamp2'][user_id]
if(max_timestamp_u_dict3['max_time_stamp3'][user_id]==lagtime_mean3):
lagtime3[i]=lagtime_mean3
else:
lagtime3[i]=timestamp-max_timestamp_u_dict3['max_time_stamp3'][user_id]
max_timestamp_u_dict3['max_time_stamp3'][user_id]=max_timestamp_u_dict2['max_time_stamp2'][user_id]
max_timestamp_u_dict2['max_time_stamp2'][user_id]=max_timestamp_u_dict['max_time_stamp'][user_id]
max_timestamp_u_dict['max_time_stamp'][user_id]=timestamp
else:
lagtime[i]=lagtime_mean
max_timestamp_u_dict['max_time_stamp'].update({user_id:timestamp})
 lagtime2[i]=lagtime_mean2
max_timestamp_u_dict2['max_time_stamp2'].update({user_id:lagtime_mean2})
 lagtime3[i]=lagtime_mean3
max_timestamp_u_dict3['max_time_stamp3'].update({user_id:lagtime_mean3})
 if user_id in user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].keys() :
delta_prior_question_elapsed_time[i]=prior_question_elapsed_time-user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]=prior_question_elapsed_time
else:
delta_prior_question_elapsed_time[i]=delta_prior_question_elapsed_time_mean
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].update({user_id:prior_question_elapsed_time})
 i=i+1
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df=test_df.merge(questions_df.loc[questions_df.index.isin(test_df['content_id'])],
how='left', on='content_id', right_index=True)
 test_df['user_lecture_lv'] = user_lecture_sum / user_lecture_count
test_df['user_lecture_sum'] = user_lecture_sum
test_df['user_interaction_count'] = user_lecture_count
test_df['user_interaction_timestamp_mean'] = test_df['timestamp']/user_lecture_count
test_df['user_correctness'] = user_sum / user_count
test_df['user_uncorrect_count'] =user_count-user_sum
test_df['user_correct_count'] =user_sum
test_df['task_container_correctness'] = task_container_sum / task_container_count
test_df['task_container_cor_count'] = task_container_sum
test_df['task_container_uncor_count'] =task_container_count-task_container_sum
test_df['task_container_std'] = task_container_std
test_df['explanation_mean'] = explanation_sum / explanation_count
test_df['explanation_true_count'] = explanation_sum
test_df['explanation_false_count'] = explanation_count-explanation_sum
test_df['delta_prior_question_elapsed_time'] = delta_prior_question_elapsed_time
test_df["attempt_no"] = test_df[["user_id", "content_id"]].apply(lambda row: get_max_attempt(row["user_id"], row["content_id"]), axis=1)
 test_df["lagtime"]=lagtime
test_df["lagtime2"]=lagtime2
test_df["lagtime3"]=lagtime3
test_df['timestamp']=test_df['timestamp']/(1000*3600)
 test_df.timestamp=test_df.timestamp.astype('float16')
 test_df['lagtime']=test_df['lagtime']/(1000*3600)
 test_df.lagtime=test_df.lagtime.astype('float32')
 test_df['lagtime2']=test_df['lagtime2']/(1000*3600)
 test_df.lagtime2=test_df.lagtime2.astype('float32')
 test_df['lagtime3']=test_df['lagtime3']/(1000*3600)
 test_df.lagtime3=test_df.lagtime3.astype('float32')
 test_df['user_interaction_timestamp_mean']=test_df['user_interaction_timestamp_mean']/(1000*3600)
 test_df.user_interaction_timestamp_mean=test_df.user_interaction_timestamp_mean.astype('float32')
 test_df['user_correctness'].fillna(0.67, inplace=True)
 sub_preds = np.zeros(test_df.shape[0])
 for i, model in enumerate(clfs, 1):
test_preds = model.predict(test_df[features])
 sub_preds += test_preds
test_df[target]=sub_preds / len(clfs)
 env.predict(test_df[['row_id', target]] )