Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<count_values>
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1
print(train.Embarked.count())
 train['Embarked'].fillna(train['Embarked'].mode() [0] ,inplace=True )<count_values>
_ = np.seterr(divide='ignore', invalid='ignore' )
train.Embarked.count()<groupby>
data_types_dict = {
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id':'int8',
'task_container_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'
train[['SibSp' , 'Survived']].groupby('SibSp' ).mean()<groupby>
train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()
train[['Parch','Survived']].groupby('Parch' ).mean()<feature_engineering>
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )
train['Alone']=0
train.loc[(train['SibSp']==0)&(train['Parch']==0), 'Alone'] = 1
test['Alone']=0
test.loc[(test['SibSp']==0)&(test['Parch']==0), 'Alone'] = 1<drop_column>
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]
drop_features = ['PassengerId' , 'Name' , 'SibSp' , 'Parch' , 'Ticket' , 'Cabin']
train.drop(drop_features , axis=1, inplace = True)
 test.drop(drop_features , axis=1 , inplace = True )<data_type_conversions>
train_lectures = train_df[train_df.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )
def map_all(frame):
frame['Sex'] = frame.Sex.map({'female': 0 , 'male': 1} ).astype(int)
 frame['Embarked'] = frame.Embarked.map({'S' : 0 , 'C': 1 , 'Q':2} ).astype(int)
 frame.loc[frame.Age <= 16 , 'Age'] = 0
frame.loc[(frame.Age >16)&(frame.Age<=32), 'Age'] = 1
frame.loc[(frame.Age >32)&(frame.Age<=48), 'Age'] = 2
frame.loc[(frame.Age >48)&(frame.Age<=64), 'Age'] = 3
frame.loc[(frame.Age >64)&(frame.Age<=80), 'Age'] = 4
frame.loc[(frame.Fare <= 7.91), 'Fare'] = 0
frame.loc[(frame.Fare > 7.91)&(frame.Fare <= 14.454), 'Fare'] = 1
frame.loc[(frame.Fare > 14.454)&(frame.Fare <= 31), 'Fare'] = 2
frame.loc[(frame.Fare > 31), 'Fare'] = 3<categorify>
user_lecture_stats_part = train_lectures.groupby('user_id',as_index = False)[part_lectures_columns + types_of_lectures_columns].sum()
map_all(train)
 train.head()<categorify>
lecturedata_types_dict = {
'user_id': 'int32',
'part_1': 'int8',
'part_2': 'int8',
'part_3': 'int8',
'part_4': 'int8',
'part_5': 'int8',
'part_6': 'int8',
'part_7': 'int8',
'type_of_concept': 'int8',
'type_of_intention': 'int8',
'type_of_solving_question': 'int8',
'type_of_starter': 'int8'
}
user_lecture_stats_part = user_lecture_stats_part.astype(lecturedata_types_dict )
map_all(test)
 test.head()<split>
for column in user_lecture_stats_part.columns:
if(column !='user_id'):
user_lecture_stats_part[column] =(user_lecture_stats_part[column] > 0 ).astype('int8' )
x_train,x_test,y_train,y_test=train_test_split(train.drop('Survived',axis=1),train.Survived,test_size=0.20,random_state=66 )<choose_model_class>
train_lectures[train_lectures.user_id==5382]
models = [LogisticRegression() ,LinearSVC() ,SVC(kernel='rbf'),KNeighborsClassifier() ,RandomForestClassifier() ,
DecisionTreeClassifier() ,GradientBoostingClassifier() ,GaussianNB() , LinearDiscriminantAnalysis() ,
QuadraticDiscriminantAnalysis() ]
model_names=['LogisticRegression','LinearSVM','rbfSVM','KNearestNeighbors','RandomForestClassifier','DecisionTree',
'GradientBoostingClassifier','GaussianNB', 'LinearDiscriminantAnalysis','QuadraticDiscriminantAnalysis']
accuracy = []
for model in range(len(models)) :
clf = models[model]
clf.fit(x_train,y_train)
 pred = clf.predict(x_test)
 accuracy.append(accuracy_score(pred , y_test))
compare = pd.DataFrame({'Algorithm' : model_names , 'Accuracy' : accuracy})
 compare<find_best_params>
user_lecture_stats_part[user_lecture_stats_part.user_id==5382]
params_dict={'criterion':['gini','entropy'],'max_depth':[5.21,5.22,5.23,5.24,5.25,5.26,5.27,5.28,5.29,5.3]}
clf_dt=GridSearchCV(estimator=DecisionTreeClassifier() ,param_grid=params_dict,scoring='accuracy', cv=5)
 clf_dt.fit(x_train,y_train)
 pred=clf_dt.predict(x_test)
 print(accuracy_score(pred,y_test))
print(clf_dt.best_params_ )<train_on_grid>
del(train_lectures )
no_of_test=[i+1 for i in range(50)]
params_dict={'n_neighbors':no_of_test}
clf_knn=GridSearchCV(estimator=KNeighborsClassifier() ,param_grid=params_dict,scoring='accuracy')
 clf_knn.fit(x_train,y_train)
 pred=clf_knn.predict(x_test)
 print(accuracy_score(pred,y_test))
print(clf_knn.best_params_ )<save_to_csv>
cum = train_df.groupby('user_id')['content_type_id'].agg(['cumsum', 'cumcount'])
 train_df['user_lecture_cumsum'] = cum['cumsum']
train_df['user_lecture_lv'] = cum['cumsum'] / cum['cumcount']
train_df.user_lecture_lv=train_df.user_lecture_lv.astype('float16')
 train_df.user_lecture_cumsum=train_df.user_lecture_cumsum.astype('int8')
 user_lecture_agg = train_df.groupby('user_id')['content_type_id'].agg(['sum', 'count'] )
pred = clf_dt.predict(test)
 d = {'PassengerId' : test_df.PassengerId , 'Survived' : pred}
answer = pd.DataFrame(d)
 answer.to_csv('Prediction.csv' , index=False )<import_modules>
train_df['prior_question_had_explanation'].fillna(False, inplace=True)
 train_df = train_df.astype(data_types_dict)
 train_df = train_df[train_df[target] != -1].reset_index(drop=True)
 prior_question_elapsed_time_mean=train_df['prior_question_elapsed_time'].mean()
train_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True )
warnings.filterwarnings('ignore')
 df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )<prepare_x_and_y>
max_timestamp_u = train_df[['user_id','timestamp']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u.columns = ['user_id', 'max_time_stamp']
y = df_train['Survived']
test_index = df_test['PassengerId']
combine = [df_train, df_test]<categorify>
train_df['lagtime'] = train_df.groupby('user_id')['timestamp'].shift()
train_df['lagtime']=train_df['timestamp']-train_df['lagtime']
train_df['lagtime'].fillna(0, inplace=True)
 train_df.lagtime=train_df.lagtime.astype('int32')
for dataset in combine:
dataset['Sex'] = dataset['Sex'].map({'female': 1, 'male': 0} ).astype(int)
 freq_port = df_train.Embarked.dropna().mode() [0]
for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].fillna(freq_port)
 for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
for dataset in combine:
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0)
 print("Values in Sex:", df_train['Sex'].unique())
 print("Values in Embarked:", df_train['Embarked'].unique())
 print("Values in Title:", sorted(df_train['Title'].unique()))<drop_column>
lagtime_agg = train_df.groupby('user_id')['lagtime'].agg(['mean'])
 train_df['lagtime_mean'] = train_df['user_id'].map(lagtime_agg['mean'])
 train_df.lagtime_mean=train_df.lagtime_mean.astype('int32' )
df_train = df_train.drop(['FareBand'], axis=1)
 combine = [df_train, df_test]<feature_engineering>
user_prior_question_elapsed_time = train_df[['user_id','prior_question_elapsed_time']].groupby(['user_id'] ).tail(1)
 user_prior_question_elapsed_time.columns = ['user_id', 'prior_question_elapsed_time']
for dataset in combine:
dataset.loc[(dataset.Age.isnull()), 'Age'] = dataset.Age.median()<sort_values>
train_df['delta_prior_question_elapsed_time'] = train_df.groupby('user_id')['prior_question_elapsed_time'].shift()
train_df['delta_prior_question_elapsed_time']=train_df['prior_question_elapsed_time']-train_df['delta_prior_question_elapsed_time']
train_df['delta_prior_question_elapsed_time'].fillna(0, inplace=True)
df_train['AgeBand'] = pd.cut(df_train['Age'], 5)
 df_train[['AgeBand', 'Survived']].groupby(['AgeBand'], as_index=False ).mean().sort_values(by='AgeBand', ascending=True )<drop_column>
train_df.delta_prior_question_elapsed_time=train_df.delta_prior_question_elapsed_time.astype('int32' )
df_train = df_train.drop(['AgeBand'], axis=1)
 combine = [df_train, df_test]
df_train.head()<feature_engineering>
train_df['timestamp']=train_df['timestamp']/(1000*3600)
 train_df.timestamp=train_df.timestamp.astype('int16')
for dataset in combine:
dataset['Age*Class'] = dataset.Age * dataset.Pclass
df_train.loc[:, ['Age*Class', 'Age', 'Pclass']].head(5 )<drop_column>
train_df['lag'] = train_df.groupby('user_id')[target].shift()
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df['user_correct_cumsum'] = cum['cumsum']
train_df['user_correct_cumcount'] = cum['cumcount']
train_df.drop(columns=['lag'], inplace=True)
 train_df['user_correct_cumsum'].fillna(0, inplace=True)
 train_df.user_correctness=train_df.user_correctness.astype('float16')
 train_df.user_correct_cumcount=train_df.user_correct_cumcount.astype('int16')
 train_df.user_correct_cumsum=train_df.user_correct_cumsum.astype('int16' )
df_train = df_train.drop(['Name', 'Ticket', 'Cabin'], axis=1)
 df_test = df_test.drop(['Name', 'Ticket', 'Cabin'], axis=1)
 combine = [df_train, df_test]
display(combine[0].head())
 display(combine[1].head() )<prepare_x_and_y>
train_df.prior_question_had_explanation=train_df.prior_question_had_explanation.astype('int8')
 train_df['lag'] = train_df.groupby('user_id')['prior_question_had_explanation'].shift()
df_train_X = combine[0].drop(["Survived", "PassengerId"], axis=1)
 train_y = combine[0]["Survived"]
test_test_X = combine[1].drop("PassengerId", axis=1 ).copy()<choose_model_class>
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 train_df['explanation_mean'] = cum['cumsum'] / cum['cumcount']
train_df['explanation_cumsum'] = cum['cumsum']
train_df.drop(columns=['lag'], inplace=True)
 train_df['explanation_mean'].fillna(0, inplace=True)
 train_df['explanation_cumsum'].fillna(0, inplace=True)
 train_df.explanation_mean=train_df.explanation_mean.astype('float16')
 train_df.explanation_cumsum=train_df.explanation_cumsum.astype('int16' )
MLA = [
ensemble.AdaBoostClassifier() ,
ensemble.BaggingClassifier() ,
ensemble.ExtraTreesClassifier() ,
ensemble.GradientBoostingClassifier() ,
ensemble.RandomForestClassifier() ,
gaussian_process.GaussianProcessClassifier() ,
linear_model.LogisticRegressionCV() ,
linear_model.PassiveAggressiveClassifier() ,
linear_model.RidgeClassifierCV() ,
linear_model.SGDClassifier() ,
linear_model.Perceptron() ,
naive_bayes.BernoulliNB() ,
naive_bayes.GaussianNB() ,
neighbors.KNeighborsClassifier() ,
svm.SVC(probability=True),
svm.NuSVC(probability=True),
svm.LinearSVC() ,
tree.DecisionTreeClassifier() ,
tree.ExtraTreeClassifier() ,
discriminant_analysis.LinearDiscriminantAnalysis() ,
discriminant_analysis.QuadraticDiscriminantAnalysis() ,
XGBClassifier()
]
cv_split = model_selection.ShuffleSplit(n_splits = 10, test_size =.3, train_size =.6, random_state = 0)
 MLA_columns = ['MLA Name', 'MLA Parameters','MLA Train Accuracy Mean', 'MLA Test Accuracy Mean', 'MLA Test Accuracy 3*STD' ,'MLA Time']
MLA_compare = pd.DataFrame(columns = MLA_columns)
 MLA_predict = {}
row_index = 0
for alg in MLA:
MLA_name = alg.__class__.__name__
MLA_compare.loc[row_index, 'MLA Name'] = MLA_name
MLA_compare.loc[row_index, 'MLA Parameters'] = str(alg.get_params())
 cv_results = model_selection.cross_validate(alg, df_train_X, train_y, cv = cv_split, return_train_score=True)
 MLA_compare.loc[row_index, 'MLA Time'] = cv_results['fit_time'].mean()
MLA_compare.loc[row_index, 'MLA Train Accuracy Mean'] = cv_results['train_score'].mean()
MLA_compare.loc[row_index, 'MLA Test Accuracy Mean'] = cv_results['test_score'].mean()
MLA_compare.loc[row_index, 'MLA Test Accuracy 3*STD'] = cv_results['test_score'].std() *3
alg.fit(df_train_X, train_y)
 MLA_compare.loc[row_index, 'F1 Score'] = metrics.f1_score(train_y, alg.predict(df_train_X))
MLA_predict[MLA_name] = alg.predict(test_test_X)
 row_index+=1
MLA_compare.sort_values(by = ['F1 Score'], ascending = False, inplace = True )<find_best_params>
del cum
gc.collect()
best_model = MLA_compare.loc[MLA_compare['F1 Score'].idxmax() ]['MLA Name']
best_model_score = round(MLA_compare.loc[MLA_compare['F1 Score'].idxmax() ]['F1 Score'],3)
 print("Best model:",best_model)
 print("F1 Score:",best_model_score )<save_to_csv>
train_df["attempt_no"] = 1
train_df.attempt_no=train_df.attempt_no.astype('int8')
 train_df["attempt_no"] = train_df[["user_id","content_id",'attempt_no']].groupby(["user_id","content_id"])["attempt_no"].cumsum()
predictions = MLA_predict[best_model]
predictions = predictions.ravel()
data_to_submit = pd.DataFrame({
'PassengerId': test_index,
'Survived': predictions
})
 data_to_submit.to_csv("results.csv", index=False )<set_options>
explanation_agg = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['sum', 'count'])
 explanation_agg=explanation_agg.astype('int16')
warnings.filterwarnings("ignore")
 %matplotlib inline<load_from_csv>
user_agg = train_df.groupby('user_id')[target].agg(['sum', 'count'])
 content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count','var'])
 task_container_agg = train_df.groupby('task_container_id')[target].agg(['sum', 'count','var'])
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv' )<count_missing_values>
user_agg=user_agg.astype('int16')
 content_agg=content_agg.astype('float32')
 task_container_agg=task_container_agg.astype('float32' )
test.isnull().sum()<count_missing_values>
attempt_no_agg=train_df.groupby(["user_id","content_id"])["attempt_no"].agg(['sum'])
 attempt_no_agg=attempt_no_agg.astype('int8')
test.isnull().sum()<count_missing_values>
train_df['content_count'] = train_df['content_id'].map(content_agg['count'] ).astype('int32')
 train_df['content_sum'] = train_df['content_id'].map(content_agg['sum'] ).astype('int32')
 train_df['content_correctness'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 train_df.content_correctness=train_df.content_correctness.astype('float16')
 train_df['task_container_sum'] = train_df['task_container_id'].map(task_container_agg['sum'] ).astype('int32')
 train_df['task_container_std'] = train_df['task_container_id'].map(task_container_agg['var'] ).astype('float16')
 train_df['task_container_correctness'] = train_df['task_container_id'].map(task_container_agg['sum'] / task_container_agg['count'])
 train_df.task_container_correctness=train_df.task_container_correctness.astype('float16' )
train.isnull().sum()<count_missing_values>
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 1,3,4],
dtype={'question_id': 'int16','bundle_id': 'int16', 'part': 'int8','tags': 'str'} 
)
 questions_df['part_bundle_id']=questions_df['part']*100000+questions_df['bundle_id']
questions_df.part_bundle_id=questions_df.part_bundle_id.astype('int32')
 tag = questions_df["tags"].str.split(" ", n = 10, expand = True)
 tag.columns = ['tags1','tags2','tags3','tags4','tags5','tags6']
tag.fillna(0, inplace=True)
 tag = tag.astype('int16')
 questions_df = pd.concat([questions_df,tag],axis=1 ).drop(['tags'],axis=1 )
train.isnull().sum()<count_values>
questions_df.rename(columns={'question_id':'content_id'}, inplace=True )
train['Survived'].value_counts(normalize=True )<groupby>
questions_df['content_correctness'] = questions_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 questions_df.content_correctness=questions_df.content_correctness.astype('float16')
 questions_df['content_correctness_std'] = questions_df['content_id'].map(content_agg['var'])
 questions_df.content_correctness_std=questions_df.content_correctness_std.astype('float16' )
train['Survived'].groupby(train['Pclass'] ).mean()<count_values>
part_agg = questions_df.groupby('part')['content_correctness'].agg(['mean', 'var'])
 questions_df['part_correctness_mean'] = questions_df['part'].map(part_agg['mean'])
 questions_df['part_correctness_std'] = questions_df['part'].map(part_agg['var'])
 questions_df.part_correctness_mean=questions_df.part_correctness_mean.astype('float16')
 questions_df.part_correctness_std=questions_df.part_correctness_std.astype('float16' )
train['Title'] = train['Title'] = train.Name.str.extract('([A-Za-z]+)\.', expand=False)
 train['Title'].value_counts()<groupby>
bundle_agg = questions_df.groupby('bundle_id')['content_correctness'].agg(['mean'])
 questions_df['bundle_correctness'] = questions_df['bundle_id'].map(bundle_agg['mean'])
 questions_df.bundle_correctness=questions_df.bundle_correctness.astype('float16' )
train['Survived'].groupby(train['Title'] ).mean()<feature_engineering>
tags1_agg = questions_df.groupby('tags1')['content_correctness'].agg(['mean', 'var'])
 questions_df['tags1_correctness_mean'] = questions_df['tags1'].map(tags1_agg['mean'])
 questions_df['tags1_correctness_std'] = questions_df['tags1'].map(tags1_agg['var'])
 questions_df.tags1_correctness_mean=questions_df.tags1_correctness_mean.astype('float16')
 questions_df.tags1_correctness_std=questions_df.tags1_correctness_std.astype('float16' )
train['Name_Len'] = train['Name'].apply(lambda x: len(x))
train['Survived'].groupby(pd.qcut(train['Name_Len'],5)).mean()<count_values>
questions_df.drop(columns=['content_correctness'], inplace=True )
pd.qcut(train['Name_Len'],5 ).value_counts()<count_values>
del bundle_agg
del part_agg
del tags1_agg
gc.collect()
train['Sex'].value_counts(normalize=True )<groupby>
train_df['user_correctness'].fillna(1, inplace=True)
 train_df['attempt_no'].fillna(1, inplace=True)
 train_df.fillna(0, inplace=True )
train['Survived'].groupby(train['Sex'] ).mean()<groupby>
features = [
'timestamp',
'lagtime',
'lagtime_mean',
'content_id',
'task_container_id',
'user_lecture_cumsum',
'user_lecture_lv',
'prior_question_elapsed_time',
'delta_prior_question_elapsed_time',
'user_correctness',
'user_correct_cumcount',
'user_correct_cumsum',
'content_correctness',
'content_correctness_std',
'content_count',
'content_sum',
'task_container_correctness',
'task_container_std',
'task_container_sum',
'bundle_correctness',
'attempt_no',
'part',
'part_correctness_mean',
'part_correctness_std',
'tags1',
'tags1_correctness_mean',
'tags1_correctness_std',
'tags2',
'tags3',
'tags4',
'tags5',
'tags6',
'bundle_id',
'part_bundle_id',
'explanation_mean',
'explanation_cumsum',
'prior_question_had_explanation',
]
categorical_columns= [
'content_id',
'task_container_id',
'part',
'tags1',
'tags2',
'tags3',
'tags4',
'tags5',
'tags6',
'bundle_id',
'part_bundle_id',
'prior_question_had_explanation',
]
train['Survived'].groupby(pd.qcut(train['Age'],5)).mean()<count_values>
flag_lgbm=True
clfs = list()
params = {
'num_leaves': 350,
'max_bin':700,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.58,
'bagging_fraction': 0.58,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.05,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'random_state': 47
}
trains=list()
valids=list()
num=1
for i in range(0,num):
train_df_clf=train_df.sample(n=1200*10000)
 print('sample end')
 del train_df
users=train_df_clf['user_id'].drop_duplicates()
users=users.sample(frac=0.025)
 users_df=pd.DataFrame()
users_df['user_id']=users.values
valid_df_newuser = pd.merge(train_df_clf, users_df, on=['user_id'], how='inner',right_index=True)
 del users_df
del users
gc.collect()
train_df_clf.drop(valid_df_newuser.index, inplace=True)
 train_df_clf = pd.merge(train_df_clf, questions_df, on='content_id', how='left',right_index=True)
 valid_df_newuser = pd.merge(valid_df_newuser, questions_df, on='content_id', how='left',right_index=True)
 valid_df=train_df_clf.sample(frac=0.15)
 train_df_clf.drop(valid_df.index, inplace=True)
 valid_df = valid_df.append(valid_df_newuser)
 del valid_df_newuser
gc.collect()
trains.append(train_df_clf)
 valids.append(valid_df)
 print('valid_df length：',len(valid_df))
pd.qcut(train['Age'],5 ).value_counts()<count_values>
del train_df_clf
del valid_df
gc.collect()
train['SibSp'].value_counts()<groupby>
for i in range(0,num):
tr_data = lgb.Dataset(trains[i][features], label=trains[i][target])
 va_data = lgb.Dataset(valids[i][features], label=valids[i][target])
 del trains
del valids
gc.collect()
model = lgb.train(
 params,
tr_data,
num_boost_round=5000,
valid_sets=[tr_data, va_data],
early_stopping_rounds=50,
feature_name=features,
categorical_feature=categorical_columns,
verbose_eval=50 
)
 clfs.append(model)
 lgb.plot_importance(model, importance_type='gain')
 plt.show()
del tr_data
del va_data
gc.collect()
train['Survived'].groupby(train['SibSp'] ).mean()<count_values>
train['Parch'].value_counts()<groupby>
train['Survived'].groupby(train['Parch'] ).mean()<count_values>
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int))
content_sum_dict = content_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
content_count_dict = content_agg['count'].astype('int32' ).to_dict(defaultdict(int))
del user_agg
del content_agg
gc.collect()
task_container_sum_dict = task_container_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
task_container_count_dict = task_container_agg['count'].astype('int32' ).to_dict(defaultdict(int))
task_container_std_dict = task_container_agg['var'].astype('float16' ).to_dict(defaultdict(int))
explanation_sum_dict = explanation_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
explanation_count_dict = explanation_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del task_container_agg
del explanation_agg
gc.collect()
train['Ticket_Len'] = train['Ticket'].apply(lambda x: len(x))
train['Ticket_Len'].value_counts()<count_values>
user_lecture_sum_dict = user_lecture_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_lecture_count_dict = user_lecture_agg['count'].astype('int16' ).to_dict(defaultdict(int))
lagtime_mean_dict = lagtime_agg['mean'].astype('int32' ).to_dict(defaultdict(int))
del user_lecture_agg
del lagtime_agg
gc.collect()
train['Ticket_Lett'] = train['Ticket'].apply(lambda x: str(x)[0])
 train['Ticket_Lett'].value_counts()<count_values>
attempt_no_agg=attempt_no_agg[attempt_no_agg['sum'] >1]
attempt_no_sum_dict = attempt_no_agg['sum'].to_dict(defaultdict(int))
del attempt_no_agg
gc.collect()
pd.qcut(train['Fare'], 3 ).value_counts()<groupby>
max_timestamp_u_dict=max_timestamp_u.set_index('user_id' ).to_dict()
user_prior_question_elapsed_time_dict=user_prior_question_elapsed_time.set_index('user_id' ).to_dict()
del max_timestamp_u
del user_prior_question_elapsed_time
gc.collect()
train['Survived'].groupby(pd.qcut(train['Fare'], 3)).mean()<count_values>
def get_max_attempt(user_id,content_id):
k =(user_id,content_id)
 if k in attempt_no_sum_dict.keys() :
attempt_no_sum_dict[k]+=1
return attempt_no_sum_dict[k]
attempt_no_sum_dict[k] = 1
return attempt_no_sum_dict[k]
train['Cabin_Lett'] = train['Cabin'].apply(lambda x: str(x)[0])
 train['Cabin_Lett'].value_counts()<groupby>
iter_test = env.iter_test()
prior_test_df = None
<feature_engineering><EOS> 
%%time
for(test_df, sample_prediction_df)in iter_test:
if prior_test_df is not None:
prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)
 prior_test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 prior_test_df.prior_question_had_explanation=prior_test_df.prior_question_had_explanation.astype('int8')
 user_ids = prior_test_df['user_id'].values
content_ids = prior_test_df['content_id'].values
task_container_ids = prior_test_df['task_container_id'].values
prior_question_had_explanations = prior_test_df['prior_question_had_explanation'].values
targets = prior_test_df[target].values
for user_id, content_id,prior_question_had_explanation,task_container_id,answered_correctly in zip(user_ids, content_ids, prior_question_had_explanations,task_container_ids,targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
explanation_sum_dict[user_id] += prior_question_had_explanation
explanation_count_dict[user_id] += 1
prior_test_df = test_df.copy()
lecture_test_df = test_df[test_df['content_type_id'] == 1].reset_index(drop=True)
 for i,(user_id,content_type_id, content_id)in enumerate(zip(lecture_test_df['user_id'].values,lecture_test_df['content_type_id'].values,lecture_test_df['content_id'].values)) :
user_lecture_sum_dict[user_id] += content_type_id
user_lecture_count_dict[user_id] += 1
if(len(user_lecture_stats_part[user_lecture_stats_part.user_id==user_id])==0):
user_lecture_stats_part = user_lecture_stats_part.append([{'user_id':user_id}], ignore_index=True)
 user_lecture_stats_part.fillna(0, inplace=True)
 user_lecture_stats_part.loc[user_lecture_stats_part.user_id==user_id,part_lectures_columns + types_of_lectures_columns]+=lectures_df[lectures_df.lecture_id==content_id][part_lectures_columns + types_of_lectures_columns].values
else:
user_lecture_stats_part.loc[user_lecture_stats_part.user_id==user_id,part_lectures_columns + types_of_lectures_columns]+=lectures_df[lectures_df.lecture_id==content_id][part_lectures_columns + types_of_lectures_columns].values
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df.prior_question_had_explanation=test_df.prior_question_had_explanation.astype('int8')
 test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)
 user_lecture_sum = np.zeros(len(test_df), dtype=np.int16)
 user_lecture_count = np.zeros(len(test_df), dtype=np.int16)
 user_sum = np.zeros(len(test_df), dtype=np.int16)
 user_count = np.zeros(len(test_df), dtype=np.int16)
 content_sum = np.zeros(len(test_df), dtype=np.int32)
 content_count = np.zeros(len(test_df), dtype=np.int32)
 task_container_sum = np.zeros(len(test_df), dtype=np.int32)
 task_container_count = np.zeros(len(test_df), dtype=np.int32)
 task_container_std = np.zeros(len(test_df), dtype=np.float16)
 content_task_mean = np.zeros(len(test_df), dtype=np.float16)
 explanation_sum = np.zeros(len(test_df), dtype=np.int32)
 explanation_count = np.zeros(len(test_df), dtype=np.int32)
 delta_prior_question_elapsed_time = np.zeros(len(test_df), dtype=np.int32)
 attempt_no_count = np.zeros(len(test_df), dtype=np.int16)
 lagtime = np.zeros(len(test_df), dtype=np.int32)
 lagtime_mean = np.zeros(len(test_df), dtype=np.int32)
 for i,(user_id,prior_question_had_explanation,content_type_id,prior_question_elapsed_time,timestamp, content_id,task_container_id)in enumerate(zip(test_df['user_id'].values,test_df['prior_question_had_explanation'].values,test_df['content_type_id'].values,test_df['prior_question_elapsed_time'].values,test_df['timestamp'].values, test_df['content_id'].values, test_df['task_container_id'].values)) :
user_lecture_sum_dict[user_id] += content_type_id
user_lecture_count_dict[user_id] += 1
user_lecture_sum[i] = user_lecture_sum_dict[user_id]
user_lecture_count[i] = user_lecture_count_dict[user_id]
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
content_sum[i] = content_sum_dict[content_id]
content_count[i] = content_count_dict[content_id]
task_container_sum[i] = task_container_sum_dict[task_container_id]
task_container_count[i] = task_container_count_dict[task_container_id]
task_container_std[i]=task_container_std_dict[task_container_id]
explanation_sum[i] = explanation_sum_dict[user_id]
explanation_count[i] = explanation_count_dict[user_id]
if user_id in max_timestamp_u_dict['max_time_stamp'].keys() :
lagtime[i]=timestamp-max_timestamp_u_dict['max_time_stamp'][user_id]
max_timestamp_u_dict['max_time_stamp'][user_id]=timestamp
lagtime_mean[i]=(lagtime_mean_dict[user_id]+lagtime[i])/2
else:
lagtime[i]=0
max_timestamp_u_dict['max_time_stamp'].update({user_id:timestamp})
 lagtime_mean_dict.update({user_id:timestamp})
 lagtime_mean[i]=(lagtime_mean_dict[user_id]+lagtime[i])/2
if user_id in user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].keys() :
delta_prior_question_elapsed_time[i]=prior_question_elapsed_time-user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]=prior_question_elapsed_time
else:
delta_prior_question_elapsed_time[i]=0
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].update({user_id:prior_question_elapsed_time})
 test_df=test_df.merge(questions_df.loc[questions_df.index.isin(test_df['content_id'])],
how='left', on='content_id', right_index=True)
 test_df['user_lecture_lv'] = user_lecture_sum / user_lecture_count
test_df['user_lecture_cumsum'] = user_lecture_sum
test_df['user_correctness'] = user_sum / user_count
test_df['user_correct_cumcount'] =user_count
test_df['user_correct_cumsum'] =user_sum
test_df['content_correctness'] = content_sum / content_count
test_df['content_count'] = content_count
test_df['content_sum'] = content_sum
test_df['task_container_correctness'] = task_container_sum / task_container_count
test_df['task_container_sum'] = task_container_sum
test_df['task_container_std'] = task_container_std
test_df['explanation_mean'] = explanation_sum / explanation_count
test_df['explanation_cumsum'] = explanation_sum
test_df['delta_prior_question_elapsed_time'] = delta_prior_question_elapsed_time
test_df["attempt_no"] = test_df[["user_id", "content_id"]].apply(lambda row: get_max_attempt(row["user_id"], row["content_id"]), axis=1)
 test_df["lagtime"]=lagtime
test_df["lagtime_mean"]=lagtime_mean
test_df['user_correctness'].fillna(1, inplace=True)
 test_df['attempt_no'].fillna(1, inplace=True)
 test_df.fillna(0, inplace=True)
 test_df['timestamp']=test_df['timestamp']/(1000*3600)
 test_df.timestamp=test_df.timestamp.astype('int16')
 sub_preds = np.zeros(test_df.shape[0])
 for i, model in enumerate(clfs, 1):
test_preds = model.predict(test_df[features])
 sub_preds += test_preds
test_df[target]=sub_preds / len(clfs)
 env.predict(test_df[['row_id', target]] )