Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<choose_model_class>
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
rms_score_cv(lasso )<choose_model_class>
_ = np.seterr(divide='ignore', invalid='ignore' )
e_net = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
rms_score_cv(e_net )<choose_model_class>
data_types_dict = {
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id':'int8',
'task_container_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'
ransac = RANSACRegressor()
rms_score_cv(ransac )<import_modules>
train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()
svm_reg = LinearSVR(epsilon=0.01)
 rms_score_cv(svm_reg )<import_modules>
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )
svm_poly_reg = SVR(kernel='poly', degree=2, C=100, epsilon=0.1)
 rms_score_cv(svm_poly_reg )<compute_train_metric>
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]
decision_tree = DecisionTreeRegressor(max_depth=7, random_state=42)
 rms_score_cv(decision_tree )<compute_train_metric>
train_lectures = train_df[train_df.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )
rand_forest = RandomForestRegressor(max_depth=15, random_state=42)
 rms_score_cv(rand_forest )<choose_model_class>
user_lecture_stats_part = train_lectures.groupby('user_id',as_index = False)[part_lectures_columns + types_of_lectures_columns].sum()
voting_reg = VotingRegressor(
 estimators=[('e_net', e_net),('ridge', ridge),('svm_poly_reg', svm_poly_reg),('rand_forest', rand_forest)])
 rms_score_cv(voting_reg )<choose_model_class>
lecturedata_types_dict = {
'user_id': 'int32',
'part_1': 'int8',
'part_2': 'int8',
'part_3': 'int8',
'part_4': 'int8',
'part_5': 'int8',
'part_6': 'int8',
'part_7': 'int8',
'type_of_concept': 'int8',
'type_of_intention': 'int8',
'type_of_solving_question': 'int8',
'type_of_starter': 'int8'
}
user_lecture_stats_part = user_lecture_stats_part.astype(lecturedata_types_dict )
bagging_reg = BaggingRegressor(e_net, n_estimators=300, bootstrap=True, n_jobs=-1)
<choose_model_class>
for column in user_lecture_stats_part.columns:
if(column !='user_id'):
user_lecture_stats_part[column] =(user_lecture_stats_part[column] > 0 ).astype('int8' )
bagging_reg = BaggingRegressor(e_net, n_estimators=500, bootstrap=False, n_jobs=-1)
<import_modules>
train_lectures[train_lectures.user_id==5382]
extra_tree = ExtraTreesRegressor(random_state=42)
 rms_score_cv(extra_tree )<choose_model_class>
user_lecture_stats_part[user_lecture_stats_part.user_id==5382]
ada_boost = AdaBoostRegressor(e_net, n_estimators=500, learning_rate=0.5)
<choose_model_class>
del(train_lectures )
gbr = GradientBoostingRegressor(max_depth=5, n_estimators=15, learning_rate=0.6)
 rms_score_cv(gbr )<split>
cum = train_df.groupby('user_id')['content_type_id'].agg(['cumsum', 'cumcount'])
 train_df['user_lecture_cumsum'] = cum['cumsum']
train_df['user_lecture_lv'] = cum['cumsum'] / cum['cumcount']
train_df.user_lecture_lv=train_df.user_lecture_lv.astype('float16')
 train_df.user_lecture_cumsum=train_df.user_lecture_cumsum.astype('int8')
 user_lecture_agg = train_df.groupby('user_id')['content_type_id'].agg(['sum', 'count'] )
estimators = [ e_net, ridge, svm_poly_reg, rand_forest, extra_tree]
slice = int(( 0.8)*(len(train_prepared)))
 train_d = train_prepared[:slice]
train_l = train_labels[:slice]
val_d = train_prepared[slice: ]
val_l = train_labels[slice:]<train_model>
train_df['prior_question_had_explanation'].fillna(False, inplace=True)
 train_df = train_df.astype(data_types_dict)
 train_df = train_df[train_df[target] != -1].reset_index(drop=True)
 prior_question_elapsed_time_mean=train_df['prior_question_elapsed_time'].mean()
train_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True )
for estimator in estimators:
estimator.fit(train_d, train_l )<predict_on_test>
max_timestamp_u = train_df[['user_id','timestamp']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u.columns = ['user_id', 'max_time_stamp']
val_preds = np.empty(( len(val_d), len(estimators)) , dtype=np.float32)
 for index, estimator in enumerate(estimators):
val_preds[:, index] = model.predict(val_d)
 blender = RandomForestRegressor(n_estimators=200, oob_score=True, random_state=42)
 blender.fit(val_preds, val_l)
 blender.oob_score_<train_model>
train_df['lagtime'] = train_df.groupby('user_id')['timestamp'].shift()
train_df['lagtime']=train_df['timestamp']-train_df['lagtime']
train_df['lagtime'].fillna(0, inplace=True)
 train_df.lagtime=train_df.lagtime.astype('int32')
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y=None):
X = X
y = y.values
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=42)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def get_metafeatures(self, X):
return np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )<choose_model_class>
lagtime_agg = train_df.groupby('user_id')['lagtime'].agg(['mean'])
 train_df['lagtime_mean'] = train_df['user_id'].map(lagtime_agg['mean'])
 train_df.lagtime_mean=train_df.lagtime_mean.astype('int32' )
stacked_averaged_models = StackingAveragedModels(base_models =(e_net, ridge, svm_poly_reg, rand_forest), meta_model = lasso)
<train_model>
user_prior_question_elapsed_time = train_df[['user_id','prior_question_elapsed_time']].groupby(['user_id'] ).tail(1)
 user_prior_question_elapsed_time.columns = ['user_id', 'prior_question_elapsed_time']
el_net = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
el_net.fit(train_prepared, train_labels )<categorify>
train_df['delta_prior_question_elapsed_time'] = train_df.groupby('user_id')['prior_question_elapsed_time'].shift()
train_df['delta_prior_question_elapsed_time']=train_df['prior_question_elapsed_time']-train_df['delta_prior_question_elapsed_time']
train_df['delta_prior_question_elapsed_time'].fillna(0, inplace=True)
test_prepared = full_pipeline.transform(test_df )<predict_on_test>
train_df.delta_prior_question_elapsed_time=train_df.delta_prior_question_elapsed_time.astype('int32' )
pred = el_net.predict(test_prepared)
 predSalePrice = [np.exp(x)for x in pred]<create_dataframe>
train_df['timestamp']=train_df['timestamp']/(1000*3600)
 train_df.timestamp=train_df.timestamp.astype('int16')
output = pd.DataFrame({
'Id' : test_df['Id'],
'SalePrice' : predSalePrice
} )<save_to_csv>
train_df['lag'] = train_df.groupby('user_id')[target].shift()
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df['user_correct_cumsum'] = cum['cumsum']
train_df['user_correct_cumcount'] = cum['cumcount']
train_df.drop(columns=['lag'], inplace=True)
 train_df['user_correct_cumsum'].fillna(0, inplace=True)
 train_df.user_correctness=train_df.user_correctness.astype('float16')
 train_df.user_correct_cumcount=train_df.user_correct_cumcount.astype('int16')
 train_df.user_correct_cumsum=train_df.user_correct_cumsum.astype('int16' )
output.to_csv('prediction.csv', index=False )<load_from_csv>
train_df.prior_question_had_explanation=train_df.prior_question_had_explanation.astype('int8')
 train_df['lag'] = train_df.groupby('user_id')['prior_question_had_explanation'].shift()
df = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<set_options>
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 train_df['explanation_mean'] = cum['cumsum'] / cum['cumcount']
train_df['explanation_cumsum'] = cum['cumsum']
train_df.drop(columns=['lag'], inplace=True)
 train_df['explanation_mean'].fillna(0, inplace=True)
 train_df['explanation_cumsum'].fillna(0, inplace=True)
 train_df.explanation_mean=train_df.explanation_mean.astype('float16')
 train_df.explanation_cumsum=train_df.explanation_cumsum.astype('int16' )
style.use('ggplot')
<drop_column>
del cum
gc.collect()
df.drop(['Name'], axis=1, inplace = True)
 df.drop(['PassengerId'], axis=1, inplace= True)
 test.drop(['Name'], axis=1, inplace = True)
<correct_missing_values>
train_df["attempt_no"] = 1
train_df.attempt_no=train_df.attempt_no.astype('int8')
 train_df["attempt_no"] = train_df[["user_id","content_id",'attempt_no']].groupby(["user_id","content_id"])["attempt_no"].cumsum()
df.fillna(0, inplace = True)
 test.fillna(0, inplace = True)
<define_variables>
explanation_agg = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['sum', 'count'])
 explanation_agg=explanation_agg.astype('int16')
bins = [0,1, 5, 10, 25, 50, 100]
labels = [1,2,3,4,5,6]<feature_engineering>
user_agg = train_df.groupby('user_id')[target].agg(['sum', 'count'])
 content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count','var'])
 task_container_agg = train_df.groupby('task_container_id')[target].agg(['sum', 'count','var'])
df['Age'] = pd.cut(df['Age'], bins = bins, labels = labels)
 test['Age'] = pd.cut(test['Age'], bins = bins, labels = labels)
<categorify>
user_agg=user_agg.astype('int16')
 content_agg=content_agg.astype('float32')
 task_container_agg=task_container_agg.astype('float32' )
df = pd.get_dummies(df, columns = ['Sex'],drop_first = True)
 test = pd.get_dummies(test, columns = ['Sex'],drop_first = True )<categorify>
attempt_no_agg=train_df.groupby(["user_id","content_id"])["attempt_no"].agg(['sum'])
 attempt_no_agg=attempt_no_agg.astype('int8')
bins = [0,10,20,30,50, 100, 200 , 250, 300, 350, 400, 450, 500, 550]
labels = [1,2,3,4,5,6,7,8,9,10,11,12,13]
df['Fare'] = pd.cut(df['Fare'], bins = bins, labels = labels)
 test['Fare'] = pd.cut(test['Fare'], bins = bins, labels = labels)
 df['Fare'] = df['Fare'].astype('int32')
 test['Fare'] = test['Fare'].astype('int32' )<normalization>
train_df['content_count'] = train_df['content_id'].map(content_agg['count'] ).astype('int32')
 train_df['content_sum'] = train_df['content_id'].map(content_agg['sum'] ).astype('int32')
 train_df['content_correctness'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 train_df.content_correctness=train_df.content_correctness.astype('float16')
 train_df['task_container_sum'] = train_df['task_container_id'].map(task_container_agg['sum'] ).astype('int32')
 train_df['task_container_std'] = train_df['task_container_id'].map(task_container_agg['var'] ).astype('float16')
 train_df['task_container_correctness'] = train_df['task_container_id'].map(task_container_agg['sum'] / task_container_agg['count'])
 train_df.task_container_correctness=train_df.task_container_correctness.astype('float16' )
fare_scale = preprocessing.MinMaxScaler()
df_fares = df['Fare'].values
scaled_fares = df_fares.reshape(-1,1)
 scaled_fares = fare_scale.fit_transform(scaled_fares)
 scaled_fares = scaled_fares.flatten()
df['Fare'] = pd.Series(scaled_fares)
 test_fares = test['Fare'].values
scaled_fares = test_fares.reshape(-1,1)
 scaled_fares = fare_scale.transform(scaled_fares)
 scaled_fares = scaled_fares.flatten()
test['Fare'] = pd.Series(scaled_fares )<categorify>
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 1,3,4],
dtype={'question_id': 'int16','bundle_id': 'int16', 'part': 'int8','tags': 'str'} 
)
 questions_df['part_bundle_id']=questions_df['part']*100000+questions_df['bundle_id']
questions_df.part_bundle_id=questions_df.part_bundle_id.astype('int32')
 tag = questions_df["tags"].str.split(" ", n = 10, expand = True)
 tag.columns = ['tags1','tags2','tags3','tags4','tags5','tags6']
tag.fillna(0, inplace=True)
 tag = tag.astype('int16')
 questions_df = pd.concat([questions_df,tag],axis=1 ).drop(['tags'],axis=1 )
label_enc = preprocessing.LabelEncoder()
df['Cabin'] = df['Cabin'].astype('str')
 test['Cabin'] = test['Cabin'].astype('str')
 enc_list = []
for i in df['Cabin'].values:
enc_list.append(i)
 for i in test['Cabin'].values:
enc_list.append(i)
 label_enc.fit(enc_list)
 df['Cabin'] = label_enc.transform(df['Cabin'])
 test['Cabin'] = label_enc.transform(test['Cabin'] )<categorify>
questions_df.rename(columns={'question_id':'content_id'}, inplace=True )
label_enc = preprocessing.LabelEncoder()
df['Embarked'] = df['Embarked'].astype('str')
 df['Embarked'] = label_enc.fit_transform(df['Embarked'])
 test['Embarked'] = test['Embarked'].astype('str')
 test['Embarked'] = label_enc.transform(test['Embarked'] )<categorify>
questions_df['content_correctness'] = questions_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 questions_df.content_correctness=questions_df.content_correctness.astype('float16')
 questions_df['content_correctness_std'] = questions_df['content_id'].map(content_agg['var'])
 questions_df.content_correctness_std=questions_df.content_correctness_std.astype('float16' )
label_enc = preprocessing.LabelEncoder()
df['Ticket'] = df['Ticket'].astype('str')
 test['Ticket'] = test['Ticket'].astype('str')
 enc_list = []
for i in df['Ticket'].values:
enc_list.append(i)
 for i in test['Ticket'].values:
enc_list.append(i)
 label_enc.fit(enc_list)
 df['Ticket'] = label_enc.transform(df['Ticket'])
 test['Ticket'] = label_enc.transform(test['Ticket'] )<data_type_conversions>
part_agg = questions_df.groupby('part')['content_correctness'].agg(['mean', 'var'])
 questions_df['part_correctness_mean'] = questions_df['part'].map(part_agg['mean'])
 questions_df['part_correctness_std'] = questions_df['part'].map(part_agg['var'])
 questions_df.part_correctness_mean=questions_df.part_correctness_mean.astype('float16')
 questions_df.part_correctness_std=questions_df.part_correctness_std.astype('float16' )
df = df.astype('float')
 df.fillna(0, inplace = True)
 test = test.astype('float')
 test.fillna(0, inplace = True )<prepare_x_and_y>
bundle_agg = questions_df.groupby('bundle_id')['content_correctness'].agg(['mean'])
 questions_df['bundle_correctness'] = questions_df['bundle_id'].map(bundle_agg['mean'])
 questions_df.bundle_correctness=questions_df.bundle_correctness.astype('float16' )
X = np.array(df.drop(['Survived'], axis = 1))
X = preprocessing.scale(X)
 Y = np.array(df['Survived'] )<split>
tags1_agg = questions_df.groupby('tags1')['content_correctness'].agg(['mean', 'var'])
 questions_df['tags1_correctness_mean'] = questions_df['tags1'].map(tags1_agg['mean'])
 questions_df['tags1_correctness_std'] = questions_df['tags1'].map(tags1_agg['var'])
 questions_df.tags1_correctness_mean=questions_df.tags1_correctness_mean.astype('float16')
 questions_df.tags1_correctness_std=questions_df.tags1_correctness_std.astype('float16' )
X_train, X_test, Y_train, Y_test = train_test_split(X,Y,stratify = Y, test_size = 0.1, random_state = 31 )<train_model>
questions_df.drop(columns=['content_correctness'], inplace=True )
model = svm.SVC(kernel = 'poly',degree=3, random_state = 31, gamma = "auto", C = 1)
 model.fit(X_train, Y_train)
 predictions = model.predict(X_train)
 accuracy = accuracy_score(predictions,Y_train)
 print("Training accuracy = %0.2f" %(accuracy * 100))<predict_on_test>
del bundle_agg
del part_agg
del tags1_agg
gc.collect()
predictions = model.predict(X_test)
 accuracy = accuracy_score(predictions,Y_test)
 print("Testing accuracy = %0.2f" %(accuracy * 100))<train_model>
train_df['user_correctness'].fillna(1, inplace=True)
 train_df['attempt_no'].fillna(1, inplace=True)
 train_df.fillna(0, inplace=True )
clf = DecisionTreeClassifier(max_depth=4, random_state = 31)
 clf.fit(X_train, Y_train)
 predictions = model.predict(X_train)
 accuracy_clf = accuracy_score(predictions, Y_train)
 print("Training accuracy = %0.2f" %(accuracy_clf * 100))<compute_test_metric>
features = [
'timestamp',
'lagtime',
'lagtime_mean',
'content_id',
'task_container_id',
'user_lecture_cumsum',
'user_lecture_lv',
'prior_question_elapsed_time',
'delta_prior_question_elapsed_time',
'user_correctness',
'user_correct_cumcount',
'user_correct_cumsum',
'content_correctness',
'content_correctness_std',
'content_count',
'content_sum',
'task_container_correctness',
'task_container_std',
'task_container_sum',
'bundle_correctness',
'attempt_no',
'part',
'part_correctness_mean',
'part_correctness_std',
'tags1',
'tags1_correctness_mean',
'tags1_correctness_std',
'tags2',
'tags3',
'tags4',
'tags5',
'tags6',
'bundle_id',
'part_bundle_id',
'explanation_mean',
'explanation_cumsum',
'prior_question_had_explanation',
]
categorical_columns= [
'content_id',
'task_container_id',
'part',
'tags1',
'tags2',
'tags3',
'tags4',
'tags5',
'tags6',
'bundle_id',
'part_bundle_id',
'prior_question_had_explanation',
]
predictions = clf.predict(X_test)
 accuracy_clf = accuracy_score(predictions, Y_test)
 print("Testing accuracy = %0.2f" %(accuracy_clf * 100))<normalization>
flag_lgbm=True
clfs = list()
params = {
'num_leaves': 350,
'max_bin':700,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.58,
'bagging_fraction': 0.58,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.05,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'random_state': 47
}
trains=list()
valids=list()
num=1
for i in range(0,num):
train_df_clf=train_df.sample(n=1200*10000)
 print('sample end')
 del train_df
users=train_df_clf['user_id'].drop_duplicates()
users=users.sample(frac=0.025)
 users_df=pd.DataFrame()
users_df['user_id']=users.values
valid_df_newuser = pd.merge(train_df_clf, users_df, on=['user_id'], how='inner',right_index=True)
 del users_df
del users
gc.collect()
train_df_clf.drop(valid_df_newuser.index, inplace=True)
 train_df_clf = pd.merge(train_df_clf, questions_df, on='content_id', how='left',right_index=True)
 valid_df_newuser = pd.merge(valid_df_newuser, questions_df, on='content_id', how='left',right_index=True)
 valid_df=train_df_clf.sample(frac=0.15)
 train_df_clf.drop(valid_df.index, inplace=True)
 valid_df = valid_df.append(valid_df_newuser)
 del valid_df_newuser
gc.collect()
trains.append(train_df_clf)
 valids.append(valid_df)
 print('valid_df length：',len(valid_df))
Pid = test['PassengerId']
X_final_test = test.drop(['PassengerId'], axis = 1)
 X_final_test = preprocessing.scale(X_final_test)
<predict_on_test>
del train_df_clf
del valid_df
gc.collect()
Y_pred = clf.predict(X_final_test )<feature_engineering>
for i in range(0,num):
tr_data = lgb.Dataset(trains[i][features], label=trains[i][target])
 va_data = lgb.Dataset(valids[i][features], label=valids[i][target])
 del trains
del valids
gc.collect()
model = lgb.train(
 params,
tr_data,
num_boost_round=5000,
valid_sets=[tr_data, va_data],
early_stopping_rounds=50,
feature_name=features,
categorical_feature=categorical_columns,
verbose_eval=50 
)
 clfs.append(model)
 lgb.plot_importance(model, importance_type='gain')
 plt.show()
del tr_data
del va_data
gc.collect()
test['Survived'] = Y_pred<data_type_conversions>
test = test.astype('int32' )<create_dataframe>
submission = pd.DataFrame({
"PassengerId": test["PassengerId"],
"Survived": test["Survived"]
} )<save_to_csv>
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int))
content_sum_dict = content_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
content_count_dict = content_agg['count'].astype('int32' ).to_dict(defaultdict(int))
del user_agg
del content_agg
gc.collect()
task_container_sum_dict = task_container_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
task_container_count_dict = task_container_agg['count'].astype('int32' ).to_dict(defaultdict(int))
task_container_std_dict = task_container_agg['var'].astype('float16' ).to_dict(defaultdict(int))
explanation_sum_dict = explanation_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
explanation_count_dict = explanation_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del task_container_agg
del explanation_agg
gc.collect()
submission.to_csv('submission.csv', index=False )<save_to_csv>
user_lecture_sum_dict = user_lecture_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_lecture_count_dict = user_lecture_agg['count'].astype('int16' ).to_dict(defaultdict(int))
lagtime_mean_dict = lagtime_agg['mean'].astype('int32' ).to_dict(defaultdict(int))
del user_lecture_agg
del lagtime_agg
gc.collect()
submission.to_csv('submission.csv', index=False )<set_options>
attempt_no_agg=attempt_no_agg[attempt_no_agg['sum'] >1]
attempt_no_sum_dict = attempt_no_agg['sum'].to_dict(defaultdict(int))
del attempt_no_agg
gc.collect()
assert sys.version_info >=(3, 5)
 assert sklearn.__version__ >= "0.20"
%matplotlib inline
warnings.filterwarnings(action="ignore" )<load_from_csv>
max_timestamp_u_dict=max_timestamp_u.set_index('user_id' ).to_dict()
user_prior_question_elapsed_time_dict=user_prior_question_elapsed_time.set_index('user_id' ).to_dict()
del max_timestamp_u
del user_prior_question_elapsed_time
gc.collect()
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 train["SalePrice"] = np.log1p(train["SalePrice"])
 print(train.shape, test.shape )<sort_values>
def get_max_attempt(user_id,content_id):
k =(user_id,content_id)
 if k in attempt_no_sum_dict.keys() :
attempt_no_sum_dict[k]+=1
return attempt_no_sum_dict[k]
attempt_no_sum_dict[k] = 1
return attempt_no_sum_dict[k]
def missing_values_calculate(trainset):
nulldata =(trainset.isnull().sum() / len(trainset)) * 100
nulldata = nulldata.drop(nulldata[nulldata == 0].index ).sort_values(ascending=False)
 ratio_missing_data = pd.DataFrame({'Ratio' : nulldata})
 return ratio_missing_data.head(30 )<count_missing_values>
iter_test = env.iter_test()
prior_test_df = None
missing_values_calculate(train )<categorify>
%%time
for(test_df, sample_prediction_df)in iter_test:
if prior_test_df is not None:
prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)
 prior_test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 prior_test_df.prior_question_had_explanation=prior_test_df.prior_question_had_explanation.astype('int8')
 user_ids = prior_test_df['user_id'].values
content_ids = prior_test_df['content_id'].values
task_container_ids = prior_test_df['task_container_id'].values
prior_question_had_explanations = prior_test_df['prior_question_had_explanation'].values
targets = prior_test_df[target].values
for user_id, content_id,prior_question_had_explanation,task_container_id,answered_correctly in zip(user_ids, content_ids, prior_question_had_explanations,task_container_ids,targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
explanation_sum_dict[user_id] += prior_question_had_explanation
explanation_count_dict[user_id] += 1
prior_test_df = test_df.copy()
lecture_test_df = test_df[test_df['content_type_id'] == 1].reset_index(drop=True)
 for i,(user_id,content_type_id, content_id)in enumerate(zip(lecture_test_df['user_id'].values,lecture_test_df['content_type_id'].values,lecture_test_df['content_id'].values)) :
user_lecture_sum_dict[user_id] += content_type_id
user_lecture_count_dict[user_id] += 1
if(len(user_lecture_stats_part[user_lecture_stats_part.user_id==user_id])==0):
user_lecture_stats_part = user_lecture_stats_part.append([{'user_id':user_id}], ignore_index=True)
 user_lecture_stats_part.fillna(0, inplace=True)
 user_lecture_stats_part.loc[user_lecture_stats_part.user_id==user_id,part_lectures_columns + types_of_lectures_columns]+=lectures_df[lectures_df.lecture_id==content_id][part_lectures_columns + types_of_lectures_columns].values
else:
user_lecture_stats_part.loc[user_lecture_stats_part.user_id==user_id,part_lectures_columns + types_of_lectures_columns]+=lectures_df[lectures_df.lecture_id==content_id][part_lectures_columns + types_of_lectures_columns].values
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df.prior_question_had_explanation=test_df.prior_question_had_explanation.astype('int8')
 test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)
 user_lecture_sum = np.zeros(len(test_df), dtype=np.int16)
 user_lecture_count = np.zeros(len(test_df), dtype=np.int16)
 user_sum = np.zeros(len(test_df), dtype=np.int16)
 user_count = np.zeros(len(test_df), dtype=np.int16)
 content_sum = np.zeros(len(test_df), dtype=np.int32)
 content_count = np.zeros(len(test_df), dtype=np.int32)
 task_container_sum = np.zeros(len(test_df), dtype=np.int32)
 task_container_count = np.zeros(len(test_df), dtype=np.int32)
 task_container_std = np.zeros(len(test_df), dtype=np.float16)
 content_task_mean = np.zeros(len(test_df), dtype=np.float16)
 explanation_sum = np.zeros(len(test_df), dtype=np.int32)
 explanation_count = np.zeros(len(test_df), dtype=np.int32)
 delta_prior_question_elapsed_time = np.zeros(len(test_df), dtype=np.int32)
 attempt_no_count = np.zeros(len(test_df), dtype=np.int16)
 lagtime = np.zeros(len(test_df), dtype=np.int32)
 lagtime_mean = np.zeros(len(test_df), dtype=np.int32)
 for i,(user_id,prior_question_had_explanation,content_type_id,prior_question_elapsed_time,timestamp, content_id,task_container_id)in enumerate(zip(test_df['user_id'].values,test_df['prior_question_had_explanation'].values,test_df['content_type_id'].values,test_df['prior_question_elapsed_time'].values,test_df['timestamp'].values, test_df['content_id'].values, test_df['task_container_id'].values)) :
user_lecture_sum_dict[user_id] += content_type_id
user_lecture_count_dict[user_id] += 1
user_lecture_sum[i] = user_lecture_sum_dict[user_id]
user_lecture_count[i] = user_lecture_count_dict[user_id]
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
content_sum[i] = content_sum_dict[content_id]
content_count[i] = content_count_dict[content_id]
task_container_sum[i] = task_container_sum_dict[task_container_id]
task_container_count[i] = task_container_count_dict[task_container_id]
task_container_std[i]=task_container_std_dict[task_container_id]
explanation_sum[i] = explanation_sum_dict[user_id]
explanation_count[i] = explanation_count_dict[user_id]
if user_id in max_timestamp_u_dict['max_time_stamp'].keys() :
lagtime[i]=timestamp-max_timestamp_u_dict['max_time_stamp'][user_id]
max_timestamp_u_dict['max_time_stamp'][user_id]=timestamp
lagtime_mean[i]=(lagtime_mean_dict[user_id]+lagtime[i])/2
else:
lagtime[i]=0
max_timestamp_u_dict['max_time_stamp'].update({user_id:timestamp})
 lagtime_mean_dict.update({user_id:timestamp})
 lagtime_mean[i]=(lagtime_mean_dict[user_id]+lagtime[i])/2
if user_id in user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].keys() :
delta_prior_question_elapsed_time[i]=prior_question_elapsed_time-user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]=prior_question_elapsed_time
else:
delta_prior_question_elapsed_time[i]=0
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].update({user_id:prior_question_elapsed_time})
 test_df=test_df.merge(questions_df.loc[questions_df.index.isin(test_df['content_id'])],
how='left', on='content_id', right_index=True)
 test_df['user_lecture_lv'] = user_lecture_sum / user_lecture_count
test_df['user_lecture_cumsum'] = user_lecture_sum
test_df['user_correctness'] = user_sum / user_count
test_df['user_correct_cumcount'] =user_count
test_df['user_correct_cumsum'] =user_sum
test_df['content_correctness'] = content_sum / content_count
test_df['content_count'] = content_count
test_df['content_sum'] = content_sum
test_df['task_container_correctness'] = task_container_sum / task_container_count
test_df['task_container_sum'] = task_container_sum
test_df['task_container_std'] = task_container_std
test_df['explanation_mean'] = explanation_sum / explanation_count
test_df['explanation_cumsum'] = explanation_sum
test_df['delta_prior_question_elapsed_time'] = delta_prior_question_elapsed_time
test_df["attempt_no"] = test_df[["user_id", "content_id"]].apply(lambda row: get_max_attempt(row["user_id"], row["content_id"]), axis=1)
 test_df["lagtime"]=lagtime
test_df["lagtime_mean"]=lagtime_mean
test_df['user_correctness'].fillna(1, inplace=True)
 test_df['attempt_no'].fillna(1, inplace=True)
 test_df.fillna(0, inplace=True)
 test_df['timestamp']=test_df['timestamp']/(1000*3600)
 test_df.timestamp=test_df.timestamp.astype('int16')
 sub_preds = np.zeros(test_df.shape[0])
 for i, model in enumerate(clfs, 1):
test_preds = model.predict(test_df[features])
 sub_preds += test_preds
test_df[target]=sub_preds / len(clfs)
 env.predict(test_df[['row_id', target]] )