Kaggle competition: Riiid Answer Correctness Prediction
df["titles"].replace(["Mr.", "Miss.", "Mrs.", "Master.","sometitle"],[0,1,2,3,4], inplace = True)
 df["titles"].astype("int64")
 test_df["titles"].replace(["Mr.", "Miss.", "Mrs.", "Master.", "sometitle"],[0,1,2,3,4], inplace = True)
 test_df["titles"].astype("int64")
 df.drop(["Name"], axis = 1, inplace = True)
 test_df.drop(["Name"], axis = 1, inplace = True )<drop_column>
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1
df.drop(["Fare","n_fam_mem","actual_fare"], axis = 1, inplace = True)
 test_df.drop(["Fare","n_fam_mem","actual_fare"], axis = 1, inplace = True )<drop_column>
_ = np.seterr(divide='ignore', invalid='ignore')
labels = df["Survived"]
data = df.drop("Survived", axis = 1 )<define_variables>
data_types_dict = {
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id':'int8',
'task_container_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'
final_clf = None
clf_names = ["Logistic Regression", "KNN(3)", "XGBoost Classifier", "Random forest classifier", "Decision Tree Classifier",
"Gradient Boosting Classifier", "Support Vector Machine"]<train_model>
print('start read train data...')
 train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()
classifiers = []
scores = []
for i in range(10):
X_train, X_test, Y_train, Y_test = train_test_split(data, labels, test_size = 0.1)
 tempscores = []
lr_clf = LogisticRegression()
lr_clf.fit(X_train, Y_train)
 tempscores.append(( lr_clf.score(X_test, Y_test)) *100)
 knn3_clf = KNeighborsClassifier(n_neighbors = 3)
 knn3_clf.fit(X_train, Y_train)
 tempscores.append(( knn3_clf.score(X_test, Y_test)) *100)
 xgbc = XGBClassifier(n_estimators=15, seed=41)
 xgbc.fit(X_train, Y_train)
 tempscores.append(( xgbc.score(X_test, Y_test)) *100)
 rf_clf = RandomForestClassifier(n_estimators = 100)
 rf_clf.fit(X_train, Y_train)
 tempscores.append(( rf_clf.score(X_test, Y_test)) *100)
 dt_clf = DecisionTreeClassifier()
dt_clf.fit(X_train, Y_train)
 tempscores.append(( dt_clf.score(X_test, Y_test)) *100)
 gb_clf = GradientBoostingClassifier()
gb_clf.fit(X_train, Y_train)
 tempscores.append(( gb_clf.score(X_test, Y_test)) *100)
 svm_clf = SVC(gamma = "scale")
 svm_clf.fit(X_train, Y_train)
 tempscores.append(( svm_clf.score(X_test, Y_test)) *100)
 scores.append(tempscores )<create_dataframe>
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv')
scores = np.array(scores)
 clfs = pd.DataFrame({"Classifier":clf_names})
 for i in range(len(scores)) :
clfs['iteration' + str(i)] = scores[i].T
means = clfs.mean(axis = 1)
 means = means.values.tolist()
clfs["Average"] = means<train_model>
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]
clfs.set_index("Classifier", inplace = True)
 print("Accuracies : ")
 clfs["Average"].head(10 )<train_model>
train_lectures = train_df[train_df.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left')
 user_lecture_stats_part = train_lectures.groupby('user_id',as_index = False)[part_lectures_columns + types_of_lectures_columns].sum()
def create_multiple() :
ensembles = []
ensemble_scores = []
for i in range(5):
X_train, X_test, Y_train, Y_test = train_test_split(data, labels, test_size = 0.07)
 svm_clf = SVC(gamma = "scale")
 svm_clf = svm_clf.fit(X_train, Y_train)
 ensemble_scores.append(( svm_clf.score(X_test, Y_test)) *100)
 ensembles.append(svm_clf)
 return ensembles, ensemble_scores
SVM_ensembles, SVM_ensemble_scores = create_multiple()<compute_test_metric>
lecturedata_types_dict = {
'user_id': 'int32',
'part_1': 'int8',
'part_2': 'int8',
'part_3': 'int8',
'part_4': 'int8',
'part_5': 'int8',
'part_6': 'int8',
'part_7': 'int8',
'type_of_concept': 'int8',
'type_of_intention': 'int8',
'type_of_solving_question': 'int8',
'type_of_starter': 'int8'
}
user_lecture_stats_part = user_lecture_stats_part.astype(lecturedata_types_dict)
def print_ensemble_score(ensemble_scores, model_name):
e_score = 0
for i in range(len(ensemble_scores)) :
e_score = e_score + ensemble_scores[i]
print("SCORE(ENSEMBLE MODELS)" +str(model_name)+ " : " + str(e_score/len(ensemble_scores)))
 return
print_ensemble_score(SVM_ensemble_scores, "SVM" )<predict_on_test>
for column in user_lecture_stats_part.columns:
if(column !='user_id'):
user_lecture_stats_part[column] =(user_lecture_stats_part[column] > 0 ).astype('int8')
 user_lecture_stats_part.dtypes
def per_model_prediction(ensembles):
test_data = test_df
predictions_ensembles = []
for clf in ensembles:
temppredictions = clf.predict(test_data)
 predictions_ensembles.append(temppredictions)
 return predictions_ensembles<categorify>
del(train_lectures)
 gc.collect()
def get_predictions_modes(predictions_ensembles):
final_predictions_list = []
for i in range(len(predictions_ensembles[0])) :
temp = [predictions_ensembles[0][i], predictions_ensembles[1][i], predictions_ensembles[2][i], predictions_ensembles[3][i], predictions_ensembles[4][i]]
final_predictions_list.append(temp)
 final_predictions_list = np.array(final_predictions_list)
 pred_modes = stats.mode(final_predictions_list, axis = 1)
 final_predictions = []
for i in pred_modes[0]:
final_predictions.append(i[0])
 return final_predictions<categorify>
user_lecture_agg = train_df.groupby('user_id')['content_type_id'].agg(['sum', 'count'])
 user_lecture_agg=user_lecture_agg.astype('int16')
SVM_predictions_ensembles = per_model_prediction(SVM_ensembles)
 SVM_final_predictions = get_predictions_modes(SVM_predictions_ensembles )<save_to_csv>
cum = train_df.groupby('user_id')['content_type_id'].agg(['cumsum', 'cumcount'])
 cum['cumcount']=cum['cumcount']+1
train_df['user_interaction_count'] = cum['cumcount']
train_df['user_interaction_timestamp_mean'] = train_df['timestamp']/cum['cumcount']
train_df['user_lecture_sum'] = cum['cumsum']
train_df['user_lecture_lv'] = cum['cumsum'] / cum['cumcount']
train_df.user_lecture_lv=train_df.user_lecture_lv.astype('float16')
 train_df.user_lecture_sum=train_df.user_lecture_sum.astype('int16')
 train_df.user_interaction_count=train_df.user_interaction_count.astype('int16')
 train_df['user_interaction_timestamp_mean']=train_df['user_interaction_timestamp_mean']/(1000*3600)
 train_df.user_interaction_timestamp_mean=train_df.user_interaction_timestamp_mean.astype('float32')
passengerid = [892 + i for i in range(len(SVM_final_predictions)) ]
sub = pd.DataFrame({'PassengerId': passengerid, 'Survived':SVM_final_predictions})
 sub.to_csv('submission.csv', index = False )<load_from_csv>
del cum
gc.collect()
train = pd.read_csv('/kaggle/input/train.csv')
 train.head()<define_variables>
train_df['prior_question_had_explanation'].fillna(False, inplace=True)
 train_df = train_df.astype(data_types_dict)
 train_df = train_df[train_df[target] != -1].reset_index(drop=True)
temp = train<drop_column>
content_explation_agg=train_df[["content_id","prior_question_had_explanation",target]].groupby(["content_id","prior_question_had_explanation"])[target].agg(['mean'])
train.drop('Cabin',inplace=True, axis=1 )<drop_column>
content_explation_agg=content_explation_agg.unstack()
content_explation_agg=content_explation_agg.reset_index()
content_explation_agg.columns = ['content_id', 'content_explation_false_mean','content_explation_true_mean']
content_explation_agg.content_id=content_explation_agg.content_id.astype('int16')
 content_explation_agg.content_explation_false_mean=content_explation_agg.content_explation_false_mean.astype('float16')
 content_explation_agg.content_explation_true_mean=content_explation_agg.content_explation_true_mean.astype('float16')
train.drop('PassengerId',inplace=True, axis=1)
 train.drop('Ticket',inplace=True, axis=1 )<categorify>
train_df["attempt_no"] = 1
train_df.attempt_no=train_df.attempt_no.astype('int8')
 attempt_no_agg=train_df.groupby(["user_id","content_id"])["attempt_no"].agg(['sum'] ).astype('int8')
 train_df["attempt_no"] = train_df[["user_id","content_id",'attempt_no']].groupby(["user_id","content_id"])["attempt_no"].cumsum()
attempt_no_agg=attempt_no_agg[attempt_no_agg['sum'] >1]
def fill_age(columns):
Age = columns[0]
Pclass = columns[1]
if pd.isnull(Age):
if Pclass == 1 :
return 37.0
elif Pclass == 2 :
return 29.0
else :
return 24.0
else:
return Age<feature_engineering>
print('start handle timestamp...')
 prior_question_elapsed_time_mean=train_df['prior_question_elapsed_time'].mean()
train_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)
train['Age'] = train[['Age','Pclass']].apply(fill_age,axis=1 )<string_transform>
max_timestamp_u = train_df[['user_id','timestamp']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u.columns = ['user_id', 'max_time_stamp']
max_timestamp_u.user_id=max_timestamp_u.user_id.astype('int32')
 train_df['lagtime'] = train_df.groupby('user_id')['timestamp'].shift()
max_timestamp_u2 = train_df[['user_id','lagtime']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u2.columns = ['user_id', 'max_time_stamp2']
max_timestamp_u2.user_id=max_timestamp_u2.user_id.astype('int32')
 train_df['lagtime']=train_df['timestamp']-train_df['lagtime']
lagtime_mean=train_df['lagtime'].mean()
train_df['lagtime'].fillna(lagtime_mean, inplace=True)
 train_df['lagtime']=train_df['lagtime']/(1000*3600)
 train_df.lagtime=train_df.lagtime.astype('float32')
names = train.Name.str.split(',')
 names2 = []
for i in range(0,891):
names2.append(names[i][1].split('.')[0] )<categorify>
train_df['lagtime2'] = train_df.groupby('user_id')['timestamp'].shift(2)
 max_timestamp_u3 = train_df[['user_id','lagtime2']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u3.columns = ['user_id', 'max_time_stamp3']
max_timestamp_u3.user_id=max_timestamp_u3.user_id.astype('int32')
 train_df['lagtime2']=train_df['timestamp']-train_df['lagtime2']
lagtime_mean2=train_df['lagtime2'].mean()
train_df['lagtime2'].fillna(lagtime_mean2, inplace=True)
 train_df['lagtime2']=train_df['lagtime2']/(1000*3600)
 train_df.lagtime2=train_df.lagtime2.astype('float32')
 train_df['lagtime3'] = train_df.groupby('user_id')['timestamp'].shift(3)
 train_df['lagtime3']=train_df['timestamp']-train_df['lagtime3']
lagtime_mean3=train_df['lagtime3'].mean()
train_df['lagtime3'].fillna(lagtime_mean3, inplace=True)
 train_df['lagtime3']=train_df['lagtime3']/(1000*3600)
 train_df.lagtime3=train_df.lagtime3.astype('float32')
namedummies = pd.get_dummies(names2,drop_first=True )<concatenate>
train_df['timestamp']=train_df['timestamp']/(1000*3600)
 train_df.timestamp=train_df.timestamp.astype('float16')
 user_prior_question_elapsed_time = train_df[['user_id','prior_question_elapsed_time']].groupby(['user_id'] ).tail(1)
 user_prior_question_elapsed_time.columns = ['user_id', 'prior_question_elapsed_time']
train_df['delta_prior_question_elapsed_time'] = train_df.groupby('user_id')['prior_question_elapsed_time'].shift()
train_df['delta_prior_question_elapsed_time']=train_df['prior_question_elapsed_time']-train_df['delta_prior_question_elapsed_time']
delta_prior_question_elapsed_time_mean=train_df['delta_prior_question_elapsed_time'].mean()
train_df['delta_prior_question_elapsed_time'].fillna(delta_prior_question_elapsed_time_mean, inplace=True)
 train_df.delta_prior_question_elapsed_time=train_df.delta_prior_question_elapsed_time.astype('int32')
 train_df['lag'] = train_df.groupby('user_id')[target].shift()
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 user_agg = train_df.groupby('user_id')['lag'].agg(['sum', 'count'] ).astype('int16')
 cum['cumsum'].fillna(0, inplace=True)
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df['user_correct_count'] = cum['cumsum']
train_df['user_uncorrect_count'] = cum['cumcount']-cum['cumsum']
train_df.drop(columns=['lag'], inplace=True)
 train_df['user_correctness'].fillna(0.67, inplace=True)
 train_df.user_correctness=train_df.user_correctness.astype('float16')
 train_df.user_correct_count=train_df.user_correct_count.astype('int16')
 train_df.user_uncorrect_count=train_df.user_uncorrect_count.astype('int16')
 del cum
gc.collect()
train = pd.concat([train,namedummies],axis=1 )<categorify>
train_df.prior_question_had_explanation=train_df.prior_question_had_explanation.astype('int8')
 explanation_agg = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['sum', 'count'])
 explanation_agg=explanation_agg.astype('int16' )
sex = pd.get_dummies(train['Sex'],drop_first=True)
 emb = pd.get_dummies(train['Embarked'],drop_first=True )<drop_column>
cum = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['cumsum', 'cumcount'])
 cum['cumcount']=cum['cumcount']+1
train_df['explanation_mean'] = cum['cumsum'] / cum['cumcount']
train_df['explanation_true_count'] = cum['cumsum']
train_df['explanation_false_count'] = cum['cumcount']-cum['cumsum']
train_df.explanation_mean=train_df.explanation_mean.astype('float16')
 train_df.explanation_true_count=train_df.explanation_true_count.astype('int16')
 train_df.explanation_false_count=train_df.explanation_false_count.astype('int16')
train.drop(['Sex','Embarked','Name'],inplace=True, axis=1 )<concatenate>
del cum
gc.collect()
train = pd.concat([train,sex,emb],axis=1);<import_modules>
content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count','var'])
 task_container_agg = train_df.groupby('task_container_id')[target].agg(['sum', 'count','var'])
 content_agg=content_agg.astype('float32')
 task_container_agg=task_container_agg.astype('float32' )
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split<split>
train_df['task_container_uncor_count'] = train_df['task_container_id'].map(task_container_agg['count']-task_container_agg['sum'] ).astype('int32')
 train_df['task_container_cor_count'] = train_df['task_container_id'].map(task_container_agg['sum'] ).astype('int32')
 train_df['task_container_std'] = train_df['task_container_id'].map(task_container_agg['var'] ).astype('float16')
 train_df['task_container_correctness'] = train_df['task_container_id'].map(task_container_agg['sum'] / task_container_agg['count'])
 train_df.task_container_correctness=train_df.task_container_correctness.astype('float16')
 content_elapsed_time_agg=train_df.groupby('content_id')['prior_question_elapsed_time'].agg(['mean'])
 content_had_explanation_agg=train_df.groupby('content_id')['prior_question_had_explanation'].agg(['mean'])
 train_df.dtypes
X_train, X_test, y_train, y_test = train_test_split(train.drop(['Fare','Survived'],axis=1), train['Survived'], test_size=0.33, random_state=42 )<train_model>
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 1,3,4],
dtype={'question_id': 'int16','bundle_id': 'int16', 'part': 'int8','tags': 'str'} 
)
 bundle_agg = questions_df.groupby('bundle_id')['question_id'].agg(['count'])
 questions_df['content_sub_bundle'] = questions_df['bundle_id'].map(bundle_agg['count'] ).astype('int8')
 questions_df['tags'].fillna('188', inplace=True)
logmodel = LogisticRegression()
logmodel.fit(X_train,y_train);<predict_on_test>
def gettags(tags,num):
tags_splits=tags.split(" ")
 result=''
for t in tags_splits:
x=int(t)
 if(x<32*(num+1)and x>=32*num):
result=result+' '+t
return result
pred = logmodel.predict(X_test )<import_modules>
for num in range(0,6):
questions_df["tags"+str(num)] = questions_df["tags"].apply(lambda row: gettags(row,num))
le = LabelEncoder()
le.fit(np.unique(questions_df['tags'+str(num)].values))
questions_df['tags'+str(num)]=questions_df[['tags'+str(num)]].apply(le.transform)
 questions_df_dict = {
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
}
questions_df = questions_df.astype(questions_df_dict)
 questions_df.drop(columns=['tags'], inplace=True)
 questions_df['part_bundle_id']=questions_df['part']*100000+questions_df['bundle_id']
questions_df.part_bundle_id=questions_df.part_bundle_id.astype('int32')
 questions_df.rename(columns={'question_id':'content_id'}, inplace=True)
 questions_df = pd.merge(questions_df, content_explation_agg, on='content_id', how='left',right_index=True)
 del content_explation_agg
questions_df['content_correctness'] = questions_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 questions_df.content_correctness=questions_df.content_correctness.astype('float16')
 questions_df['content_correctness_std'] = questions_df['content_id'].map(content_agg['var'])
 questions_df.content_correctness_std=questions_df.content_correctness_std.astype('float16')
 questions_df['content_uncorrect_count'] = questions_df['content_id'].map(content_agg['count']-content_agg['sum'] ).astype('int32')
 questions_df['content_correct_count'] = questions_df['content_id'].map(content_agg['sum'] ).astype('int32')
 questions_df['content_elapsed_time_mean'] = questions_df['content_id'].map(content_elapsed_time_agg['mean'])
 questions_df.content_elapsed_time_mean=questions_df.content_elapsed_time_mean.astype('float16')
 questions_df['content_had_explanation_mean'] = questions_df['content_id'].map(content_had_explanation_agg['mean'])
 questions_df.content_had_explanation_mean=questions_df.content_had_explanation_mean.astype('float16')
 del content_elapsed_time_agg
del content_had_explanation_agg
gc.collect()
from sklearn.metrics import roc_curve
from sklearn.metrics import roc_auc_score<load_from_csv>
part_agg = questions_df.groupby('part')['content_correctness'].agg(['mean', 'var'])
 questions_df['part_correctness_mean'] = questions_df['part'].map(part_agg['mean'])
 questions_df['part_correctness_std'] = questions_df['part'].map(part_agg['var'])
 questions_df.part_correctness_mean=questions_df.part_correctness_mean.astype('float16')
 questions_df.part_correctness_std=questions_df.part_correctness_std.astype('float16')
 part_agg = questions_df.groupby('part')['content_uncorrect_count'].agg(['sum'])
 questions_df['part_uncor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32')
 part_agg = questions_df.groupby('part')['content_correct_count'].agg(['sum'])
 questions_df['part_cor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32')
 bundle_agg = questions_df.groupby('bundle_id')['content_correctness'].agg(['mean'])
 questions_df['bundle_correctness_mean'] = questions_df['bundle_id'].map(bundle_agg['mean'])
 questions_df.bundle_correctness_mean=questions_df.bundle_correctness_mean.astype('float16')
test = pd.read_csv('.. /input/test.csv')
 temp_test = test<drop_column>
del content_agg
del bundle_agg
del part_agg
gc.collect()
pass_id = test.PassengerId
test.drop(['Cabin','PassengerId','Ticket'],inplace=True, axis=1 )<groupby>
features_dict = {
'timestamp':'float16',
'user_interaction_count':'int16',
'user_interaction_timestamp_mean':'float32',
'lagtime':'float32',
'lagtime2':'float32',
'lagtime3':'float32',
'content_id':'int16',
'task_container_id':'int16',
'user_lecture_sum':'int16',
'user_lecture_lv':'float16',
'prior_question_elapsed_time':'float32',
'delta_prior_question_elapsed_time':'int32',
'user_correctness':'float16',
'user_uncorrect_count':'int16',
'user_correct_count':'int16',
'content_correctness_std':'float16',
'content_correct_count':'int32',
'content_uncorrect_count':'int32',
'content_elapsed_time_mean':'float16',
'content_had_explanation_mean':'float16',
'content_explation_false_mean':'float16',
'content_explation_true_mean':'float16',
'task_container_correctness':'float16',
'task_container_std':'float16',
'task_container_cor_count':'int32',
'task_container_uncor_count':'int32',
'attempt_no':'int8',
'part':'int8',
'part_correctness_mean':'float16',
'part_correctness_std':'float16',
'part_uncor_count':'int32',
'part_cor_count':'int32',
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
'part_bundle_id':'int32',
'content_sub_bundle':'int8',
'prior_question_had_explanation':'int8',
'explanation_mean':'float16',
'explanation_false_count':'int16',
'explanation_true_count':'int16',
}
categorical_columns= [
'content_id',
'task_container_id',
'part',
'tags0',
'tags1',
'tags2',
'tags3',
'tags4',
'tags5',
'part_bundle_id',
'content_sub_bundle',
'prior_question_had_explanation',
]
features=list(features_dict.keys())
print("The mean before is ", test.Age.mean())
 print(test.groupby('Pclass' ).mean() ['Age'] )<categorify>
flag_lgbm=True
clfs = list()
params = {
'num_leaves': 200,
'max_bin':450,
'feature_fraction': 0.52,
'bagging_fraction': 0.52,
'objective': 'binary',
'learning_rate': 0.05,
"boosting_type": "gbdt",
"metric": 'auc',
}
trains=list()
valids=list()
num=1
for i in range(0,num):
train_df_clf=train_df[1200*10000:2*1200*10000]
print('sample end')
 del train_df
users=train_df_clf['user_id'].drop_duplicates()
users=users.sample(frac=0.08)
 users_df=pd.DataFrame()
users_df['user_id']=users.values
valid_df_newuser = pd.merge(train_df_clf, users_df, on=['user_id'], how='inner',right_index=True)
 del users_df
del users
gc.collect()
train_df_clf.drop(valid_df_newuser.index, inplace=True)
 print('pd.merge(train_df_clf, questions_df)')
 train_df_clf = pd.merge(train_df_clf, questions_df, on='content_id', how='left',right_index=True)
 valid_df_newuser = pd.merge(valid_df_newuser, questions_df, on='content_id', how='left',right_index=True)
 print('valid_df')
 valid_df=train_df_clf.sample(frac=0.1)
 train_df_clf.drop(valid_df.index, inplace=True)
 valid_df = valid_df.append(valid_df_newuser)
 del valid_df_newuser
gc.collect()
trains.append(train_df_clf)
 valids.append(valid_df)
 print('train_df_clf length：',len(train_df_clf))
print('valid_df length：',len(valid_df))
def age_fill(columns):
Age = columns[0]
Pclass = columns[1]
if pd.isnull(Age):
if Pclass == 1 :
return 40.69
elif Pclass == 2 :
return 28.83
else :
return 24.39
else:
return Age<feature_engineering>
del train_df_clf
del valid_df
gc.collect()
test['Age'] = test[['Age','Pclass']].apply(age_fill,axis=1 )<string_transform>
for i in range(0,num):
X_train_np = trains[i][features].values.astype(np.float32)
 X_valid_np = valids[i][features].values.astype(np.float32)
 tr_data = lgb.Dataset(X_train_np, label=trains[i][target], feature_name=list(features))
va_data = lgb.Dataset(X_valid_np, label=valids[i][target], feature_name=list(features))
del trains
del valids
del X_train_np
del X_valid_np
gc.collect()
model = lgb.train(
 params,
tr_data,
num_boost_round=5000,
valid_sets=[tr_data, va_data],
early_stopping_rounds=50,
feature_name=features,
categorical_feature=categorical_columns,
verbose_eval=50 
)
 clfs.append(model)
 fig,ax = plt.subplots(figsize=(15,15))
lgb.plot_importance(model, ax=ax,importance_type='gain',max_num_features=50)
 plt.show()
del tr_data
del va_data
gc.collect()
namesx = test.Name.str.split(',')
 namesx2 = []
for i in range(0,test.shape[0]):
namesx2.append(namesx[i][1].split('.')[0] )<categorify>
import joblib
namexdummies = pd.get_dummies(namesx2,drop_first=True )<concatenate>
MAX_SEQ = 100
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):
super(SAKTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.embedding = nn.Embedding(2*n_skill+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = x + pos_x
e = self.e_embedding(question_ids)
 x = x.permute(1, 0, 2)
 e = e.permute(1, 0, 2)
 att_mask = future_mask(x.size(0)).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
skills = joblib.load("/kaggle/input/skillspklzip/skills.pkl.zip")
 n_skill = len(skills)
 group = joblib.load("/kaggle/input/muku222/group.pkl.zip")
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 nn_model = SAKTModel(n_skill, embed_dim=128)
 try:
nn_model.load_state_dict(torch.load("/kaggle/input/123456/sakt_model.pt"))
except:
nn_model.load_state_dict(torch.load("/kaggle/input/123456/sakt_model.pt", map_location='cpu'))
nn_model.to(device)
 nn_model.eval()
test = pd.concat([test,namexdummies],axis=1 )<categorify>
del user_agg
gc.collect()
task_container_sum_dict = task_container_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
task_container_count_dict = task_container_agg['count'].astype('int32' ).to_dict(defaultdict(int))
task_container_std_dict = task_container_agg['var'].astype('float16' ).to_dict(defaultdict(int))
explanation_sum_dict = explanation_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
explanation_count_dict = explanation_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del task_container_agg
del explanation_agg
gc.collect()
sex1 = pd.get_dummies(test['Sex'],drop_first=True)
 emb1 = pd.get_dummies(test['Embarked'],drop_first=True)
 test.drop(['Sex','Embarked','Name'],inplace=True, axis=1)
 test = pd.concat([test,sex1,emb1],axis=1);<correct_missing_values>
user_lecture_sum_dict = user_lecture_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_lecture_count_dict = user_lecture_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del user_lecture_agg
gc.collect()
test.fillna(test['Fare'].median() ,inplace=True )<drop_column>
max_timestamp_u_dict=max_timestamp_u.set_index('user_id' ).to_dict()
max_timestamp_u_dict2=max_timestamp_u2.set_index('user_id' ).to_dict()
max_timestamp_u_dict3=max_timestamp_u3.set_index('user_id' ).to_dict()
user_prior_question_elapsed_time_dict=user_prior_question_elapsed_time.set_index('user_id' ).to_dict()
del max_timestamp_u
del max_timestamp_u2
del max_timestamp_u3
del user_prior_question_elapsed_time
gc.collect()
new_train = train.drop([' Jonkheer',' the Countess',' Mme',' Mlle',' Major',' Lady',' Col', ' Don', ' Sir'],axis=1)
 new_test = test.drop([' Dona'],axis=1 )<split>
attempt_no_sum_dict = attempt_no_agg['sum'].to_dict(defaultdict(int))
del attempt_no_agg
gc.collect()
X_train1, X_test1, y_train1, y_test1 = train_test_split(new_train.drop(['Survived'],axis=1), new_train['Survived'], test_size=0.33, random_state=42 )<train_model>
def get_max_attempt(user_id,content_id):
k =(user_id,content_id)
 if k in attempt_no_sum_dict.keys() :
attempt_no_sum_dict[k]+=1
return attempt_no_sum_dict[k]
attempt_no_sum_dict[k] = 1
return attempt_no_sum_dict[k]
new_logmodel = LogisticRegression()
new_logmodel.fit(X_train1,y_train1 )<predict_on_test>
env = riiideducation.make_env()
iter_test = env.iter_test()
prior_test_df = None
prev_test_df1 = None
N=[0.4,0.6]
<create_dataframe><EOS> 
%%time
for(test_df, sample_prediction_df)in iter_test:
test_df1=test_df.copy()
if(prev_test_df1 is not None):
prev_test_df1['answered_correctly'] = eval(test_df1['prior_group_answers_correct'].iloc[0])
 prev_test_df1 = prev_test_df1[prev_test_df1.content_type_id == False]
prev_group = prev_test_df1[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
if prev_user_id in group.index:
group[prev_user_id] =(
 np.append(group[prev_user_id][0], prev_group[prev_user_id][0])[-MAX_SEQ:],
np.append(group[prev_user_id][1], prev_group[prev_user_id][1])[-MAX_SEQ:] 
)
 else:
group[prev_user_id] =(
 prev_group[prev_user_id][0],
prev_group[prev_user_id][1] 
)
 prev_test_df1 = test_df1.copy()
test_df1 = test_df1[test_df1.content_type_id == False]
test_dataset = TestDataset(group, test_df1, skills)
 test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)
 outs = []
for item in test_dataloader:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = nn_model(x, target_id)
 outs.extend(torch.sigmoid(output)[:, -1].view(-1 ).data.cpu().numpy())
 if prior_test_df is not None:
prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)
 prior_test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 prior_test_df.prior_question_had_explanation=prior_test_df.prior_question_had_explanation.astype('int8')
 user_ids = prior_test_df['user_id'].values
targets = prior_test_df[target].values
for user_id, answered_correctly in zip(user_ids,targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
prior_test_df = test_df.copy()
question_len=len(test_df[test_df['content_type_id'] == 0])
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df.prior_question_had_explanation=test_df.prior_question_had_explanation.astype('int8')
 test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)
 user_lecture_sum = np.zeros(question_len, dtype=np.int16)
 user_lecture_count = np.zeros(question_len, dtype=np.int16)
 user_sum = np.zeros(question_len, dtype=np.int16)
 user_count = np.zeros(question_len, dtype=np.int16)
 task_container_sum = np.zeros(question_len, dtype=np.int32)
 task_container_count = np.zeros(question_len, dtype=np.int32)
 task_container_std = np.zeros(question_len, dtype=np.float16)
 explanation_sum = np.zeros(question_len, dtype=np.int32)
 explanation_count = np.zeros(question_len, dtype=np.int32)
 delta_prior_question_elapsed_time = np.zeros(question_len, dtype=np.int32)
 attempt_no_count = np.zeros(question_len, dtype=np.int16)
 lagtime = np.zeros(question_len, dtype=np.float32)
 lagtime2 = np.zeros(question_len, dtype=np.float32)
 lagtime3 = np.zeros(question_len, dtype=np.float32)
 i=0
for j,(user_id,prior_question_had_explanation,content_type_id,prior_question_elapsed_time,timestamp, content_id,task_container_id)in enumerate(zip(test_df['user_id'].values,test_df['prior_question_had_explanation'].values,test_df['content_type_id'].values,test_df['prior_question_elapsed_time'].values,test_df['timestamp'].values, test_df['content_id'].values, test_df['task_container_id'].values)) :
user_lecture_sum_dict[user_id] += content_type_id
user_lecture_count_dict[user_id] += 1
if(content_type_id==1):
x=1
if(content_type_id==0):
user_lecture_sum[i] = user_lecture_sum_dict[user_id]
user_lecture_count[i] = user_lecture_count_dict[user_id]
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
task_container_sum[i] = task_container_sum_dict[task_container_id]
task_container_count[i] = task_container_count_dict[task_container_id]
task_container_std[i]=task_container_std_dict[task_container_id]
explanation_sum_dict[user_id] += prior_question_had_explanation
explanation_count_dict[user_id] += 1
explanation_sum[i] = explanation_sum_dict[user_id]
explanation_count[i] = explanation_count_dict[user_id]
if user_id in max_timestamp_u_dict['max_time_stamp'].keys() :
lagtime[i]=timestamp-max_timestamp_u_dict['max_time_stamp'][user_id]
if(max_timestamp_u_dict2['max_time_stamp2'][user_id]==lagtime_mean2):
lagtime2[i]=lagtime_mean2
lagtime3[i]=lagtime_mean3
else:
lagtime2[i]=timestamp-max_timestamp_u_dict2['max_time_stamp2'][user_id]
if(max_timestamp_u_dict3['max_time_stamp3'][user_id]==lagtime_mean3):
lagtime3[i]=lagtime_mean3
else:
lagtime3[i]=timestamp-max_timestamp_u_dict3['max_time_stamp3'][user_id]
max_timestamp_u_dict3['max_time_stamp3'][user_id]=max_timestamp_u_dict2['max_time_stamp2'][user_id]
max_timestamp_u_dict2['max_time_stamp2'][user_id]=max_timestamp_u_dict['max_time_stamp'][user_id]
max_timestamp_u_dict['max_time_stamp'][user_id]=timestamp
else:
lagtime[i]=lagtime_mean
max_timestamp_u_dict['max_time_stamp'].update({user_id:timestamp})
 lagtime2[i]=lagtime_mean2
max_timestamp_u_dict2['max_time_stamp2'].update({user_id:lagtime_mean2})
 lagtime3[i]=lagtime_mean3
max_timestamp_u_dict3['max_time_stamp3'].update({user_id:lagtime_mean3})
 if user_id in user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].keys() :
delta_prior_question_elapsed_time[i]=prior_question_elapsed_time-user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]=prior_question_elapsed_time
else:
delta_prior_question_elapsed_time[i]=delta_prior_question_elapsed_time_mean
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].update({user_id:prior_question_elapsed_time})
 i=i+1
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df=test_df.merge(questions_df.loc[questions_df.index.isin(test_df['content_id'])],
how='left', on='content_id', right_index=True)
 test_df['user_lecture_lv'] = user_lecture_sum / user_lecture_count
test_df['user_lecture_sum'] = user_lecture_sum
test_df['user_interaction_count'] = user_lecture_count
test_df['user_interaction_timestamp_mean'] = test_df['timestamp']/user_lecture_count
test_df['user_correctness'] = user_sum / user_count
test_df['user_uncorrect_count'] =user_count-user_sum
test_df['user_correct_count'] =user_sum
test_df['task_container_correctness'] = task_container_sum / task_container_count
test_df['task_container_cor_count'] = task_container_sum
test_df['task_container_uncor_count'] =task_container_count-task_container_sum
test_df['task_container_std'] = task_container_std
test_df['explanation_mean'] = explanation_sum / explanation_count
test_df['explanation_true_count'] = explanation_sum
test_df['explanation_false_count'] = explanation_count-explanation_sum
test_df['delta_prior_question_elapsed_time'] = delta_prior_question_elapsed_time
test_df["attempt_no"] = test_df[["user_id", "content_id"]].apply(lambda row: get_max_attempt(row["user_id"], row["content_id"]), axis=1)
 test_df["lagtime"]=lagtime
test_df["lagtime2"]=lagtime2
test_df["lagtime3"]=lagtime3
test_df['timestamp']=test_df['timestamp']/(1000*3600)
 test_df.timestamp=test_df.timestamp.astype('float16')
 test_df['lagtime']=test_df['lagtime']/(1000*3600)
 test_df.lagtime=test_df.lagtime.astype('float32')
 test_df['lagtime2']=test_df['lagtime2']/(1000*3600)
 test_df.lagtime2=test_df.lagtime2.astype('float32')
 test_df['lagtime3']=test_df['lagtime3']/(1000*3600)
 test_df.lagtime3=test_df.lagtime3.astype('float32')
 test_df['user_interaction_timestamp_mean']=test_df['user_interaction_timestamp_mean']/(1000*3600)
 test_df.user_interaction_timestamp_mean=test_df.user_interaction_timestamp_mean.astype('float32')
 test_df['user_correctness'].fillna(0.67, inplace=True)
 sub_preds = np.zeros(test_df.shape[0])
 for i, model in enumerate(clfs, 1):
test_preds = model.predict(test_df[features])
 sub_preds += test_preds
o2=sub_preds / len(clfs)
 test_df[target]=0.5*np.array(outs)+0.5*np.array(o2)
 env.predict(test_df[['row_id', target]])