Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<compute_test_metric>
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1
confusion_m(best_et, 'Confusion matrix for ExtraTreesClassifier')
 feature_importance(best_et.feature_importances_, 'Feature importance for ExtraTreesClassifier' )<compute_train_metric>
import numpy as np
import random
import pandas as pd
import joblib
vc = VotingClassifier(estimators=[('lr', best_lr),('xgb', best_xgb),('rf', best_rf),('et', best_et),('cb', best_cb)], voting='soft')
 vc.fit(X_train, y_train)
 predict_vc = vc.predict(X_valid)
 auc_vc = cross_val_score(vc, X_valid, y_valid, scoring = 'roc_auc', cv = 5, n_jobs = -1)
 acc_vc = cross_val_score(vc, X_valid, y_valid, scoring = 'accuracy', cv = 5, n_jobs = -1)
 print('AUC-ROC for ensemble models on validation dataset:', sum(auc_vc)/len(auc_vc))
print('Accuracy for ensemble models on validation dataset:', sum(acc_vc)/len(acc_vc))<compute_test_metric>
_ = np.seterr(divide='ignore', invalid='ignore' )
confusion_m(vc, 'Confusion matrix for VotingClassifier' )<compute_test_metric>
data_types_dict = {
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id':'int8',
'task_container_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'
models = ['logistic_regression', 'decision_tree', 'random_forest',
'xgboost', 'catboost',
'extra_trees', 'voting']
dict_values = {'accuracy': [acc_lr.mean() , acc_dt.mean() , acc_rf.mean() ,
acc_xgb.mean() , acc_cb.mean() ,
acc_et.mean() , acc_vc.mean() ],
'auc_roc': [auc_lr.mean() , auc_dt.mean() , auc_rf.mean() ,
auc_xgb.mean() , auc_cb.mean() ,
auc_et.mean() , auc_vc.mean() ]}
df_score = pd.DataFrame(dict_values, index = models, columns = ['accuracy', 'auc_roc'])
 df_score<split>
print('start read train data...')
 train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()
X_train = X_train.values
test = test.values
ntrain = X_train.shape[0]
ntest = test.shape[0]
n_folds = 5
kf = KFold(n_splits = n_folds, random_state = 12345)
 def get_oof(clf, x_train, y_train, x_test):
oof_train = np.zeros(( ntrain,))
oof_test = np.zeros(( ntest,))
oof_test_skf = np.empty(( n_folds, ntest))
for i,(train_index, test_index)in enumerate(kf.split(x_train)) :
x_tr = x_train[train_index]
y_tr = y_train.iloc[train_index]
x_te = x_train[test_index]
clf.fit(x_tr, y_tr)
 oof_train[test_index] = clf.predict(x_te)
 oof_test_skf[i, :] = clf.predict(x_test)
 oof_test[:] = oof_test_skf.mean(axis=0)
 return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1 )<categorify>
print('start handle lecture data...' )
et_oof_train, et_oof_test = get_oof(best_et, X_train, y_train, test)
 rf_oof_train, rf_oof_test = get_oof(best_rf,X_train, y_train, test)
 cb_oof_train, cb_oof_test = get_oof(best_cb, X_train, y_train, test)
 xgb_oof_train, xgb_oof_test = get_oof(best_xgb,X_train, y_train, test)
 lr_oof_train, lr_oof_test = get_oof(best_lr,X_train, y_train, test )<concatenate>
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )
X_train = np.concatenate(( et_oof_train, rf_oof_train, cb_oof_train, xgb_oof_train, lr_oof_train), axis=1)
 X_test = np.concatenate(( et_oof_test, rf_oof_test, cb_oof_test, xgb_oof_test, lr_oof_test), axis=1 )<train_model>
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]
best_model = XGBClassifier(n_estimators= 2000, max_depth= 4, min_child_weight= 2, gamma=0.9,
subsample=0.8, colsample_bytree=0.8, objective= 'binary:logistic',
nthread= -1,scale_pos_weight=1 ).fit(X_train, y_train )<predict_on_test>
train_lectures = train_df[train_df.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )
y_test = best_model.predict(X_test )<save_to_csv>
user_lecture_stats_part = train_lectures.groupby('user_id',as_index = False)[part_lectures_columns + types_of_lectures_columns].sum()
submission = pd.DataFrame({'PassengerId': list(range(892, 1310)) , 'Survived': y_test})
 submission['Survived'] = submission['Survived'].astype(int)
 submission.to_csv('submission.csv', index=False)
 print(submission )<set_options>
lecturedata_types_dict = {
'user_id': 'int32',
'part_1': 'int8',
'part_2': 'int8',
'part_3': 'int8',
'part_4': 'int8',
'part_5': 'int8',
'part_6': 'int8',
'part_7': 'int8',
'type_of_concept': 'int8',
'type_of_intention': 'int8',
'type_of_solving_question': 'int8',
'type_of_starter': 'int8'
}
user_lecture_stats_part = user_lecture_stats_part.astype(lecturedata_types_dict )
sns.set(style="darkgrid")
 warnings.filterwarnings('ignore')
 SEED = 42<load_from_csv>
for column in user_lecture_stats_part.columns:
if(column !='user_id'):
user_lecture_stats_part[column] =(user_lecture_stats_part[column] > 0 ).astype('int8' )
train=pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 test=pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv" )<drop_column>
del(train_lectures)
 gc.collect()
df_all = train.append(test, ignore_index=True)
 train_idx = len(train)
 test_idx = len(df_all)- len(test)
 train_ID = train['Id']
test_ID = test['Id']
train.drop("Id", axis = 1, inplace = True)
 test.drop("Id", axis = 1, inplace = True)
 df_all.drop("Id", axis = 1, inplace = True)
 print("
The train data size after dropping Id feature is : {} ".format(train.shape))
print("The test data size after dropping Id feature is : {} ".format(test.shape))<sort_values>
user_lecture_agg = train_df.groupby('user_id')['content_type_id'].agg(['sum', 'count'])
 user_lecture_agg=user_lecture_agg.astype('int16' )
def missing_percentage(df):

 total = df.isnull().sum().sort_values(ascending = False)[df.isnull().sum().sort_values(ascending = False)!= 0]
percent = round(df.isnull().sum().sort_values(ascending = False)/len(df)*100,2)[round(df.isnull().sum().sort_values(ascending = False)/len(df)*100,2)!= 0]
return pd.concat([total, percent], axis=1, keys=['Total','Percent'])
 missing_percentage(df_all )<categorify>
cum = train_df.groupby('user_id')['content_type_id'].agg(['cumsum', 'cumcount'])
 cum['cumcount']=cum['cumcount']+1
train_df['user_interaction_count'] = cum['cumcount']
train_df['user_interaction_timestamp_mean'] = train_df['timestamp']/cum['cumcount']
train_df['user_lecture_sum'] = cum['cumsum']
train_df['user_lecture_lv'] = cum['cumsum'] / cum['cumcount']
train_df.user_lecture_lv=train_df.user_lecture_lv.astype('float16')
 train_df.user_lecture_sum=train_df.user_lecture_sum.astype('int16')
 train_df.user_interaction_count=train_df.user_interaction_count.astype('int16')
 train_df['user_interaction_timestamp_mean']=train_df['user_interaction_timestamp_mean']/(1000*3600)
 train_df.user_interaction_timestamp_mean=train_df.user_interaction_timestamp_mean.astype('float32')
df_all['MSZoning'] = df_all.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
df_all["LotFrontage"] = df_all.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))<data_type_conversions>
del cum
gc.collect()
for col in(
 'PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu', 'MasVnrType',
'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond',
'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2' 
):
df_all[col] = df_all[col].fillna("None" )<data_type_conversions>
print('start handle train_df...' )
for col in(
 'LotFrontage', 'MasVnrArea',
'GarageYrBlt', 'GarageArea', 'GarageCars',
'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath', 
):
df_all[col] = df_all[col].fillna(0 )<categorify>
train_df['prior_question_had_explanation'].fillna(False, inplace=True)
 train_df = train_df.astype(data_types_dict)
 train_df = train_df[train_df[target] != -1].reset_index(drop=True )
df_all['Electrical'] = df_all['Electrical'].fillna(df_all['Electrical'].mode() [0])
 df_all['KitchenQual'] = df_all['KitchenQual'].fillna(df_all['KitchenQual'].mode() [0])
 df_all['Exterior1st'] = df_all['Exterior1st'].fillna(df_all['Exterior1st'].mode() [0])
 df_all['Exterior2nd'] = df_all['Exterior2nd'].fillna(df_all['Exterior2nd'].mode() [0])
 df_all['SaleType'] = df_all['SaleType'].fillna(df_all['SaleType'].mode() [0])
 df_all['Functional'] = df_all['Functional'].fillna(df_all['Functional'].mode() [0] )<drop_column>
content_explation_agg=train_df[["content_id","prior_question_had_explanation",target]].groupby(["content_id","prior_question_had_explanation"])[target].agg(['mean'] )
df_all = df_all.drop(['Utilities'], axis=1 )<count_missing_values>
content_explation_agg=content_explation_agg.unstack()
content_explation_agg=content_explation_agg.reset_index()
content_explation_agg.columns = ['content_id', 'content_explation_false_mean','content_explation_true_mean']
missing_percentage(df_all )<feature_engineering>
content_explation_agg.content_id=content_explation_agg.content_id.astype('int16')
 content_explation_agg.content_explation_false_mean=content_explation_agg.content_explation_false_mean.astype('float16')
 content_explation_agg.content_explation_true_mean=content_explation_agg.content_explation_true_mean.astype('float16' )
df_all['TotalPorchArea'] = df_all['OpenPorchSF'] + df_all['EnclosedPorch'] + df_all['3SsnPorch'] + df_all['ScreenPorch']
df_all['TotalSF'] = df_all['TotalBsmtSF'] + df_all['GrLivArea']
df_all['TotalBath'] = df_all['BsmtFullBath'] + df_all['BsmtHalfBath'] + df_all['FullBath'] + df_all['HalfBath']
def garage_age(row):
if row['GarageYrBlt'] == 0:
return 0
return(row['YrSold'] - row['GarageYrBlt'])
 df_all['GarageAge'] = df_all.apply(garage_age, axis=1)
 df_all['haspool'] = df_all['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 df_all['has2ndfloor'] = df_all['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 df_all['hasgarage'] = df_all['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 df_all['hasbsmt'] = df_all['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 df_all['hasfireplace'] = df_all['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )<feature_engineering>
print('start handle attempt_no...' )
df_all['HouseAge'] = df_all['YrSold'] - df_all['YearBuilt']
def remodelled(row):
if row['YearRemodAdd'] > row['YearBuilt']:
return 1
return 0
df_all['Remodelled'] = df_all.apply(remodelled, axis=1)
 def last_modelled(row):
year = max(row['YearBuilt'], row['YearRemodAdd'])
 return row['YrSold'] - year
df_all['LastModelled'] = df_all.apply(last_modelled, axis=1 )<data_type_conversions>
train_df["attempt_no"] = 1
train_df.attempt_no=train_df.attempt_no.astype('int8')
 attempt_no_agg=train_df.groupby(["user_id","content_id"])["attempt_no"].agg(['sum'] ).astype('int8')
 train_df["attempt_no"] = train_df[["user_id","content_id",'attempt_no']].groupby(["user_id","content_id"])["attempt_no"].cumsum()
df_all['MSSubClass'] = df_all['MSSubClass'].astype(str)
 df_all['YrSold'] = df_all['YrSold'].astype(str)
 df_all['MoSold'] = df_all['MoSold'].astype(str )<categorify>
print('start handle timestamp...')
 prior_question_elapsed_time_mean=train_df['prior_question_elapsed_time'].mean()
train_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True )
label_features = ['FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold']
for feature in label_features:
df_all[feature] = LabelEncoder().fit_transform(df_all[feature] )<categorify>
max_timestamp_u = train_df[['user_id','timestamp']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u.columns = ['user_id', 'max_time_stamp']
max_timestamp_u.user_id=max_timestamp_u.user_id.astype('int32' )
cat_features = []
for i in df_all.columns:
if df_all[i].dtype == "object":
if i not in label_features:
cat_features.append(i)
 encoded_features = []
for feature in cat_features:
encoded_feat = OneHotEncoder().fit_transform(df_all[feature].values.reshape(-1, 1)).toarray()
n = df_all[feature].nunique()
cols = ['{}_{}'.format(feature, n)for n in range(1, n + 1)]
encoded_df = pd.DataFrame(encoded_feat, columns=cols)
 encoded_df.index = df_all.index
encoded_features.append(encoded_df)
 len_cat = len(cat_features)
 df_all = pd.concat([df_all, *encoded_features[:len_cat]], axis=1 )<drop_column>
train_df['lagtime'] = train_df.groupby('user_id')['timestamp'].shift()
max_timestamp_u2 = train_df[['user_id','lagtime']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u2.columns = ['user_id', 'max_time_stamp2']
max_timestamp_u2.user_id=max_timestamp_u2.user_id.astype('int32' )
df_all.drop(cat_features, axis=1, inplace=True )<prepare_x_and_y>
train_df['lagtime']=train_df['timestamp']-train_df['lagtime']
lagtime_mean=train_df['lagtime'].mean()
train_df['lagtime'].fillna(lagtime_mean, inplace=True )
X = df_all[:train_idx].drop(['SalePrice'], axis=1)
 y = df_all['SalePrice'][:train_idx]
X_test = df_all[train_idx:].drop(['SalePrice'], axis=1 )<import_modules>
train_df['lagtime']=train_df['lagtime']/(1000*3600)
 train_df.lagtime=train_df.lagtime.astype('float32' )
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb<compute_train_metric>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, X.values, y, scoring="neg_mean_squared_error", cv = kf))
return(rmse )<normalization>
train_df['lagtime2'] = train_df.groupby('user_id')['timestamp'].shift(2)
 max_timestamp_u3 = train_df[['user_id','lagtime2']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u3.columns = ['user_id', 'max_time_stamp3']
max_timestamp_u3.user_id=max_timestamp_u3.user_id.astype('int32')
 train_df['lagtime2']=train_df['timestamp']-train_df['lagtime2']
lagtime_mean2=train_df['lagtime2'].mean()
train_df['lagtime2'].fillna(lagtime_mean2, inplace=True)
transformer = RobustScaler().fit(X)
 Xt = transformer.transform(X)
 Xt_test = transformer.transform(X_test )<compute_train_metric>
train_df['lagtime2']=train_df['lagtime2']/(1000*3600)
 train_df.lagtime2=train_df.lagtime2.astype('float32' )
max_evals = 10
space={
'alpha': hp.uniform("alpha", 0.0001, 0.1),
'max_iter': hp.uniform("max_iter", 200, 2000),
}
def hyperparameter_tuning(space):
model=Lasso(
 alpha =(space['alpha']),
max_iter =int(space['max_iter']),
random_state = 1 
)
 scores = cross_val_score(model, Xt, y, scoring='neg_mean_squared_error', cv=KFold(n_splits=5,shuffle=True))
print("SCORE: %0.4f" %(scores.mean()))
return {'loss':-scores.mean() , 'status': STATUS_OK }
trials = Trials()
best = fmin(fn=hyperparameter_tuning,
space=space,
algo=tpe.suggest,
max_evals=max_evals,
trials=trials)
 print(best )<compute_test_metric>
train_df['lagtime3'] = train_df.groupby('user_id')['timestamp'].shift(3)
 train_df['lagtime3']=train_df['timestamp']-train_df['lagtime3']
lagtime_mean3=train_df['lagtime3'].mean()
train_df['lagtime3'].fillna(lagtime_mean3, inplace=True)
 train_df['lagtime3']=train_df['lagtime3']/(1000*3600)
 train_df.lagtime3=train_df.lagtime3.astype('float32' )
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
score = rmsle_cv(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_train_metric>
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_train_metric>
train_df['timestamp']=train_df['timestamp']/(1000*3600)
 train_df.timestamp=train_df.timestamp.astype('float16' )
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5)
 score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<choose_model_class>
train_df['delta_prior_question_elapsed_time'] = train_df.groupby('user_id')['prior_question_elapsed_time'].shift()
train_df['delta_prior_question_elapsed_time']=train_df['prior_question_elapsed_time']-train_df['delta_prior_question_elapsed_time']
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213,
random_state =7, nthread = -1)
 score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<choose_model_class>
delta_prior_question_elapsed_time_mean=train_df['delta_prior_question_elapsed_time'].mean()
train_df['delta_prior_question_elapsed_time'].fillna(delta_prior_question_elapsed_time_mean, inplace=True)
 train_df.delta_prior_question_elapsed_time=train_df.delta_prior_question_elapsed_time.astype('int32' )
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_train_metric>
train_df['lag'] = train_df.groupby('user_id')[target].shift()
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 user_agg = train_df.groupby('user_id')['lag'].agg(['sum', 'count'] ).astype('int16')
 cum['cumsum'].fillna(0, inplace=True)
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df['user_correct_count'] = cum['cumsum']
train_df['user_uncorrect_count'] = cum['cumcount']-cum['cumsum']
train_df.drop(columns=['lag'], inplace=True)
 train_df['user_correctness'].fillna(0.67, inplace=True)
 train_df.user_correctness=train_df.user_correctness.astype('float16')
 train_df.user_correct_count=train_df.user_correct_count.astype('int16')
 train_df.user_uncorrect_count=train_df.user_uncorrect_count.astype('int16')
clf = RandomForestRegressor(n_estimators = 100, 
)
 scores = cross_val_score(clf, X, y, cv=5, scoring='neg_root_mean_squared_error')
 print(scores.mean() )<predict_on_test>
del cum
gc.collect()
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )<choose_model_class>
averaged_models = AveragingModels(models =(
 lasso,
ENet,
GBoost,
model_lgb,
model_xgb 
))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<train_model>
train_df.prior_question_had_explanation=train_df.prior_question_had_explanation.astype('int8')
 explanation_agg = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['sum', 'count'])
 explanation_agg=explanation_agg.astype('int16')
averaged_models.fit(X, y)
 av_prices = np.exp(averaged_models.predict(X_test))<save_to_csv>
cum = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['cumsum', 'cumcount'])
 cum['cumcount']=cum['cumcount']+1
train_df['explanation_mean'] = cum['cumsum'] / cum['cumcount']
train_df['explanation_true_count'] = cum['cumsum']
train_df['explanation_false_count'] = cum['cumcount']-cum['cumsum']
train_df.explanation_mean=train_df.explanation_mean.astype('float16')
 train_df.explanation_true_count=train_df.explanation_true_count.astype('int16')
 train_df.explanation_false_count=train_df.explanation_false_count.astype('int16' )
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = av_prices
os.chdir('/kaggle/working')
 sub.to_csv('av_submission.csv',index=False )<split>
del cum
gc.collect()
X_train, X_t, y_train, y_t = train_test_split(X, y, test_size=0.33, random_state=42 )<train_model>
content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count','var'])
 task_container_agg = train_df.groupby('task_container_id')[target].agg(['sum', 'count','var'])
 content_agg=content_agg.astype('float32')
 task_container_agg=task_container_agg.astype('float32' )
lasso_b = lasso.fit(X_train, y_train)
 enet_b = ENet.fit(X_train, y_train)
 gb_b = GBoost.fit(X_train, y_train)
 xgb_b = model_xgb.fit(X_train, y_train)
 lgb_b = model_lgb.fit(X_train, y_train )<predict_on_test>
train_df['task_container_uncor_count'] = train_df['task_container_id'].map(task_container_agg['count']-task_container_agg['sum'] ).astype('int32')
 train_df['task_container_cor_count'] = train_df['task_container_id'].map(task_container_agg['sum'] ).astype('int32')
 train_df['task_container_std'] = train_df['task_container_id'].map(task_container_agg['var'] ).astype('float16')
 train_df['task_container_correctness'] = train_df['task_container_id'].map(task_container_agg['sum'] / task_container_agg['count'])
 train_df.task_container_correctness=train_df.task_container_correctness.astype('float16' )
y_pred_lasso = pd.DataFrame(lasso_b.predict(X_t))
y_pred_enet = pd.DataFrame(enet_b.predict(X_t))
y_pred_gb = pd.DataFrame(gb_b.predict(X_t))
y_pred_xgb = pd.DataFrame(xgb_b.predict(X_t))
y_pred_lgb = pd.DataFrame(lgb_b.predict(X_t))<prepare_output>
content_elapsed_time_agg=train_df.groupby('content_id')['prior_question_elapsed_time'].agg(['mean'])
 content_had_explanation_agg=train_df.groupby('content_id')['prior_question_had_explanation'].agg(['mean'] )
y_pred=pd.DataFrame()
best_score = 100
blend_increments = [0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1]
best_increments = []
for i in blend_increments:
for j in blend_increments:
for k in blend_increments:
for l in blend_increments:
for m in blend_increments:
if(i+j+k+l)> 0:
y_pred['SalePrice'] =(i * y_pred_lasso[0] + j * y_pred_enet[0] + k * y_pred_gb[0] + l * y_pred_xgb[0] + m * y_pred_lgb[0])
 y_pred['SalePrice'] = y_pred['SalePrice'] /(i+j+k+l+m)
 rms = np.sqrt(mean_squared_error(y_pred['SalePrice'], y_t))
if rms < best_score:
best_score = rms
best_increments = [i, j, k, l, m]
print(best_score)
 print(best_increments)
<train_model>
print('start questions data...' )
lasso_b = lasso.fit(X, y)
 enet_b = ENet.fit(X, y)
 gb_b = GBoost.fit(X, y)
 xgb_b = model_xgb.fit(X, y)
 lgb_b = model_lgb.fit(X, y )<predict_on_test>
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 1,3,4],
dtype={'question_id': 'int16','bundle_id': 'int16', 'part': 'int8','tags': 'str'}
)
X_test = df_all[train_idx:].drop(['SalePrice'], axis=1)
 y_pred_lasso = pd.DataFrame(lasso_b.predict(X_test))
y_pred_enet = pd.DataFrame(enet_b.predict(X_test))
y_pred_gb = pd.DataFrame(gb_b.predict(X_test))
y_pred_xgb = pd.DataFrame(xgb_b.predict(X_test))
y_pred_lgb = pd.DataFrame(lgb_b.predict(X_test))<create_dataframe>
bundle_agg = questions_df.groupby('bundle_id')['question_id'].agg(['count'] )
y_pred = pd.DataFrame()<feature_engineering>
questions_df['content_sub_bundle'] = questions_df['bundle_id'].map(bundle_agg['count'] ).astype('int8' )
y_pred['SalePrice'] = np.exp(( 0.476 * y_pred_lasso[0] + 0 * y_pred_enet[0] + 0.381 * y_pred_gb[0] + 0.143 * y_pred_xgb[0] + 0 * y_pred_lgb[0]))<feature_engineering>
questions_df['tags'].fillna('188', inplace=True )
y_pred['Id'] = test_ID<save_to_csv>
def gettags(tags,num):
tags_splits=tags.split(" ")
 result=''
for t in tags_splits:
x=int(t)
 if(x<32*(num+1)and x>=32*num):
result=result+' '+t
return result
os.chdir('/kaggle/working')
 y_pred.to_csv('blend_submission.csv',index=False )<compute_train_metric>
for num in range(0,6):
questions_df["tags"+str(num)] = questions_df["tags"].apply(lambda row: gettags(row,num))
le = LabelEncoder()
le.fit(np.unique(questions_df['tags'+str(num)].values))
questions_df['tags'+str(num)]=questions_df[['tags'+str(num)]].apply(le.transform )
estimators = [ 
('lasso', lasso), 
('ENet', ENet), 
('model_gb', GBoost), 
('model_lgb', model_lgb), 
('model_xgb', model_xgb),
]
reg = StackingRegressor(
 estimators=estimators,
final_estimator=model_lgb,
passthrough = True 
)
 score = rmsle_cv(reg)
 print("Ensemble score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<train_model>
questions_df_dict = {
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
}
questions_df = questions_df.astype(questions_df_dict )
reg.fit(X, y)
 stack_prices = np.exp(reg.predict(X_test))<save_to_csv>
questions_df.drop(columns=['tags'], inplace=True )
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = stack_prices
os.chdir('/kaggle/working')
 sub.to_csv('stack_submission.csv',index=False )<set_options>
questions_df['part_bundle_id']=questions_df['part']*100000+questions_df['bundle_id']
questions_df.part_bundle_id=questions_df.part_bundle_id.astype('int32')
%matplotlib inline<load_from_csv>
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 y_sample = pd.read_csv("/kaggle/input/titanic/gender_submission.csv" )<groupby>
questions_df.rename(columns={'question_id':'content_id'}, inplace=True )
train_data.groupby('Survived' ).mean()<groupby>
questions_df = pd.merge(questions_df, content_explation_agg, on='content_id', how='left',right_index=True)
train_data.groupby(train_data['Age'].isnull() ).mean()<groupby>
del content_explation_agg
train_data.groupby('Pclass' ).mean()<groupby>
questions_df['content_correctness'] = questions_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 questions_df.content_correctness=questions_df.content_correctness.astype('float16')
 questions_df['content_correctness_std'] = questions_df['content_id'].map(content_agg['var'])
 questions_df.content_correctness_std=questions_df.content_correctness_std.astype('float16')
 questions_df['content_uncorrect_count'] = questions_df['content_id'].map(content_agg['count']-content_agg['sum'] ).astype('int32')
 questions_df['content_correct_count'] = questions_df['content_id'].map(content_agg['sum'] ).astype('int32' )
train_data.groupby('Parch' ).mean()<feature_engineering>
questions_df['content_elapsed_time_mean'] = questions_df['content_id'].map(content_elapsed_time_agg['mean'])
 questions_df.content_elapsed_time_mean=questions_df.content_elapsed_time_mean.astype('float16')
 questions_df['content_had_explanation_mean'] = questions_df['content_id'].map(content_had_explanation_agg['mean'])
 questions_df.content_had_explanation_mean=questions_df.content_had_explanation_mean.astype('float16' )
train_data['Age'] = train_data[['Age','Parch']].apply(age_aprox, axis = 1)
 test_data['Age'] = test_data[['Age','Parch']].apply(age_aprox, axis = 1 )<drop_column>
del content_elapsed_time_agg
del content_had_explanation_agg
gc.collect()
train_data['Family_count'] = train_data['SibSp'] +train_data['Parch'] 
<drop_column>
part_agg = questions_df.groupby('part')['content_correctness'].agg(['mean', 'var'])
 questions_df['part_correctness_mean'] = questions_df['part'].map(part_agg['mean'])
 questions_df['part_correctness_std'] = questions_df['part'].map(part_agg['var'])
 questions_df.part_correctness_mean=questions_df.part_correctness_mean.astype('float16')
 questions_df.part_correctness_std=questions_df.part_correctness_std.astype('float16' )
test_data['Family_count'] = test_data['SibSp'] +test_data['Parch'] 
<feature_engineering>
part_agg = questions_df.groupby('part')['content_uncorrect_count'].agg(['sum'])
 questions_df['part_uncor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32')
 part_agg = questions_df.groupby('part')['content_correct_count'].agg(['sum'])
 questions_df['part_cor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32' )
round(( train_data['Cabin'].isnull().sum() /len(train_data)) *100,2 )<create_dataframe>
bundle_agg = questions_df.groupby('bundle_id')['content_correctness'].agg(['mean'])
 questions_df['bundle_correctness_mean'] = questions_df['bundle_id'].map(bundle_agg['mean'])
 questions_df.bundle_correctness_mean=questions_df.bundle_correctness_mean.astype('float16')
train_data.pivot_table('Survived',index = 'Sex',columns = 'Embarked',aggfunc= 'count' )<feature_engineering>
for name in train_data['Name']:
train_data['Title'] = train_data['Name'].str.extract('([A-Za-z]+)\.',expand=True)
 test_data['Title'] = test_data['Name'].str.extract('([A-Za-z]+)\.',expand=True )<groupby>
del content_agg
del bundle_agg
del part_agg
gc.collect()
train_data.groupby('Title' ).count() ['PassengerId']<groupby>
features_dict = {
'user_interaction_count':'int16',
'user_interaction_timestamp_mean':'float32',
'lagtime':'float32',
'lagtime2':'float32',
'lagtime3':'float32',
'user_lecture_sum':'int16',
'user_lecture_lv':'float16',
'prior_question_elapsed_time':'float32',
'delta_prior_question_elapsed_time':'int32',
'user_correctness':'float16',
'user_correct_count':'int16',
'content_correctness':'float16',
'content_correct_count':'int32',
'content_elapsed_time_mean':'float16',
'content_had_explanation_mean':'float16',
'content_explation_false_mean':'float16',
'content_explation_true_mean':'float16',
'task_container_correctness':'float16',
'attempt_no':'int8',
'part':'int8',
'part_correctness_mean':'float16',
'tags1': 'int8',
'bundle_id':'int16',
'explanation_mean':'float16',
}
categorical_columns= [
'part',
'tags1',
'bundle_id',
]
features=list(features_dict.keys())
test_data.groupby('Title' ).count() ['PassengerId']<categorify>
flag_lgbm=True
clfs = list()
params = {
'num_leaves': 400,
'max_bin':500,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.52,
'bagging_fraction': 0.52,
'objective': 'binary',
'learning_rate': 0.2,
"boosting_type": "gbdt",
"metric": 'auc',
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
}
trains=list()
valids=list()
num=1
for i in range(0,num):
train_df_clf=train_df[1200*10000:2*1200*10000]
print('sample end')
 del train_df
users=train_df_clf['user_id'].drop_duplicates()
users=users.sample(frac=0.08)
 users_df=pd.DataFrame()
users_df['user_id']=users.values
valid_df_newuser = pd.merge(train_df_clf, users_df, on=['user_id'], how='inner',right_index=True)
 del users_df
del users
gc.collect()
train_df_clf.drop(valid_df_newuser.index, inplace=True)
 print('pd.merge(train_df_clf, questions_df)')
 train_df_clf = pd.merge(train_df_clf, questions_df, on='content_id', how='left',right_index=True)
 valid_df_newuser = pd.merge(valid_df_newuser, questions_df, on='content_id', how='left',right_index=True)
 print('valid_df')
 valid_df=train_df_clf.sample(frac=0.1)
 train_df_clf.drop(valid_df.index, inplace=True)
 valid_df = valid_df.append(valid_df_newuser)
 del valid_df_newuser
gc.collect()
trains.append(train_df_clf)
 valids.append(valid_df)
 print('train_df_clf length：',len(train_df_clf))
print('valid_df length：',len(valid_df))
gender = {'male':1,'female': 0}
train_data['Sex'] = train_data['Sex'].map(gender)
 test_data['Sex'] = test_data['Sex'].map(gender )<define_variables>
del train_df_clf
del valid_df
gc.collect()
title = {'Capt':'Others','Col':'Others','Countess':'Others','Don':'Others', 'Dr':'Others','Jonkheer':'Others', 'Lady':'Others', 'Major':'Others',
'Mlle':'Others', 'Mme':'Others', 'Ms':'Miss','Rev': 'Others','Sir':'Others','Dona': 'Others'}<categorify>
for i in range(0,num):
X_train_np = trains[i][features].values.astype(np.float32)
 X_valid_np = valids[i][features].values.astype(np.float32)
 tr_data = lgb.Dataset(X_train_np, label=trains[i][target], feature_name=list(features))
va_data = lgb.Dataset(X_valid_np, label=valids[i][target], feature_name=list(features))
del trains
del valids
del X_train_np
del X_valid_np
gc.collect()
model = lgb.train(
 params,
tr_data,
num_boost_round=5000,
valid_sets=[tr_data, va_data],
early_stopping_rounds=50,
feature_name=features,
categorical_feature=categorical_columns,
verbose_eval=50 
)
 clfs.append(model)
 fig,ax = plt.subplots(figsize=(15,15))
lgb.plot_importance(model, ax=ax,importance_type='gain',max_num_features=50)
 plt.show()
del tr_data
del va_data
gc.collect()
train_data.replace({'Title':title},inplace=True)
 test_data.replace({'Title':title},inplace=True )<groupby>
MAX_SEQ = 100
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):
super(SAKTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.embedding = nn.Embedding(2*n_skill+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = x + pos_x
e = self.e_embedding(question_ids)
 x = x.permute(1, 0, 2)
 e = e.permute(1, 0, 2)
 att_mask = future_mask(x.size(0)).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
skills = joblib.load("/kaggle/input/riiid-sakt-model-dataset-public/skills.pkl.zip")
 n_skill = len(skills)
 group = joblib.load("/kaggle/input/riiid-sakt-model-dataset-public/group.pkl.zip")
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 nn_model = SAKTModel(n_skill, embed_dim=128)
 try:
nn_model.load_state_dict(torch.load("/kaggle/input/riiid-sakt-model-dataset-public/sakt_model.pt"))
except:
nn_model.load_state_dict(torch.load("/kaggle/input/riiid-sakt-model-dataset-public/sakt_model.pt", map_location='cpu'))
nn_model.to(device)
 nn_model.eval()
train_data.groupby('Title' ).count() ['PassengerId']<prepare_x_and_y>
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int))
features = ["Pclass", "Sex", "SibSp","Parch","Age","Embarked"]
y = train_data["Survived"]
X = pd.get_dummies(train_data[features])
 titanic_test = pd.get_dummies(test_data[features])
 print(X.shape)
 print(y.shape)
<import_modules>
del user_agg
gc.collect()
task_container_sum_dict = task_container_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
task_container_count_dict = task_container_agg['count'].astype('int32' ).to_dict(defaultdict(int))
task_container_std_dict = task_container_agg['var'].astype('float16' ).to_dict(defaultdict(int))
explanation_sum_dict = explanation_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
explanation_count_dict = explanation_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del task_container_agg
del explanation_agg
gc.collect()
from sklearn.model_selection import train_test_split, cross_val_score<split>
user_lecture_sum_dict = user_lecture_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_lecture_count_dict = user_lecture_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del user_lecture_agg
gc.collect()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, random_state=42)
 X_test, X_val, y_test, y_val = train_test_split(X_test, y_test, test_size=0.5, random_state=42 )<define_variables>
max_timestamp_u_dict=max_timestamp_u.set_index('user_id' ).to_dict()
max_timestamp_u_dict2=max_timestamp_u2.set_index('user_id' ).to_dict()
max_timestamp_u_dict3=max_timestamp_u3.set_index('user_id' ).to_dict()
user_prior_question_elapsed_time_dict=user_prior_question_elapsed_time.set_index('user_id' ).to_dict()
del max_timestamp_u
del max_timestamp_u2
del max_timestamp_u3
del user_prior_question_elapsed_time
gc.collect()
for dataset in [y_train,y_val,y_test]:
print(round(len(dataset)/len(y),2))<import_modules>
attempt_no_sum_dict = attempt_no_agg['sum'].to_dict(defaultdict(int))
del attempt_no_agg
gc.collect()
from sklearn.model_selection import cross_val_score<compute_train_metric>
def get_max_attempt(user_id,content_id):
k =(user_id,content_id)
 if k in attempt_no_sum_dict.keys() :
attempt_no_sum_dict[k]+=1
return attempt_no_sum_dict[k]
attempt_no_sum_dict[k] = 1
return attempt_no_sum_dict[k]
rf_basic = RandomForestClassifier()
scores = cross_val_score(rf_basic,X_train,y_train, cv = 5)
 scores<find_best_params>
print('Parameters currently in use:
')
 pprint(rf_basic.get_params() )<import_modules>
iter_test = env.iter_test()
prior_test_df = None
prev_test_df1 = None
from sklearn.model_selection import GridSearchCV<import_modules>
N=[0.4,0.6]
<find_best_params><EOS> 
%%time
for(test_df, sample_prediction_df)in iter_test:
test_df1=test_df.copy()
if(prev_test_df1 is not None):
prev_test_df1['answered_correctly'] = eval(test_df1['prior_group_answers_correct'].iloc[0])
 prev_test_df1 = prev_test_df1[prev_test_df1.content_type_id == False]
prev_group = prev_test_df1[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
if prev_user_id in group.index:
group[prev_user_id] =(
 np.append(group[prev_user_id][0], prev_group[prev_user_id][0])[-MAX_SEQ:],
np.append(group[prev_user_id][1], prev_group[prev_user_id][1])[-MAX_SEQ:] 
)
 else:
group[prev_user_id] =(
 prev_group[prev_user_id][0],
prev_group[prev_user_id][1] 
)
 prev_test_df1 = test_df1.copy()
test_df1 = test_df1[test_df1.content_type_id == False]
test_dataset = TestDataset(group, test_df1, skills)
 test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)
 outs = []
for item in test_dataloader:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = nn_model(x, target_id)
 outs.extend(torch.sigmoid(output)[:, -1].view(-1 ).data.cpu().numpy())
 if prior_test_df is not None:
prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)
 prior_test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 prior_test_df.prior_question_had_explanation=prior_test_df.prior_question_had_explanation.astype('int8')
 user_ids = prior_test_df['user_id'].values
targets = prior_test_df[target].values
for user_id, answered_correctly in zip(user_ids,targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
prior_test_df = test_df.copy()
question_len=len(test_df[test_df['content_type_id'] == 0])
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df.prior_question_had_explanation=test_df.prior_question_had_explanation.astype('int8')
 test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)
 user_lecture_sum = np.zeros(question_len, dtype=np.int16)
 user_lecture_count = np.zeros(question_len, dtype=np.int16)
 user_sum = np.zeros(question_len, dtype=np.int16)
 user_count = np.zeros(question_len, dtype=np.int16)
 task_container_sum = np.zeros(question_len, dtype=np.int32)
 task_container_count = np.zeros(question_len, dtype=np.int32)
 task_container_std = np.zeros(question_len, dtype=np.float16)
 explanation_sum = np.zeros(question_len, dtype=np.int32)
 explanation_count = np.zeros(question_len, dtype=np.int32)
 delta_prior_question_elapsed_time = np.zeros(question_len, dtype=np.int32)
 attempt_no_count = np.zeros(question_len, dtype=np.int16)
 lagtime = np.zeros(question_len, dtype=np.float32)
 lagtime2 = np.zeros(question_len, dtype=np.float32)
 lagtime3 = np.zeros(question_len, dtype=np.float32)
 i=0
for j,(user_id,prior_question_had_explanation,content_type_id,prior_question_elapsed_time,timestamp, content_id,task_container_id)in enumerate(zip(test_df['user_id'].values,test_df['prior_question_had_explanation'].values,test_df['content_type_id'].values,test_df['prior_question_elapsed_time'].values,test_df['timestamp'].values, test_df['content_id'].values, test_df['task_container_id'].values)) :
user_lecture_sum_dict[user_id] += content_type_id
user_lecture_count_dict[user_id] += 1
if(content_type_id==1):
x=1
if(content_type_id==0):
user_lecture_sum[i] = user_lecture_sum_dict[user_id]
user_lecture_count[i] = user_lecture_count_dict[user_id]
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
task_container_sum[i] = task_container_sum_dict[task_container_id]
task_container_count[i] = task_container_count_dict[task_container_id]
task_container_std[i]=task_container_std_dict[task_container_id]
explanation_sum_dict[user_id] += prior_question_had_explanation
explanation_count_dict[user_id] += 1
explanation_sum[i] = explanation_sum_dict[user_id]
explanation_count[i] = explanation_count_dict[user_id]
if user_id in max_timestamp_u_dict['max_time_stamp'].keys() :
lagtime[i]=timestamp-max_timestamp_u_dict['max_time_stamp'][user_id]
if(max_timestamp_u_dict2['max_time_stamp2'][user_id]==lagtime_mean2):
lagtime2[i]=lagtime_mean2
lagtime3[i]=lagtime_mean3
else:
lagtime2[i]=timestamp-max_timestamp_u_dict2['max_time_stamp2'][user_id]
if(max_timestamp_u_dict3['max_time_stamp3'][user_id]==lagtime_mean3):
lagtime3[i]=lagtime_mean3
else:
lagtime3[i]=timestamp-max_timestamp_u_dict3['max_time_stamp3'][user_id]
max_timestamp_u_dict3['max_time_stamp3'][user_id]=max_timestamp_u_dict2['max_time_stamp2'][user_id]
max_timestamp_u_dict2['max_time_stamp2'][user_id]=max_timestamp_u_dict['max_time_stamp'][user_id]
max_timestamp_u_dict['max_time_stamp'][user_id]=timestamp
else:
lagtime[i]=lagtime_mean
max_timestamp_u_dict['max_time_stamp'].update({user_id:timestamp})
 lagtime2[i]=lagtime_mean2
max_timestamp_u_dict2['max_time_stamp2'].update({user_id:lagtime_mean2})
 lagtime3[i]=lagtime_mean3
max_timestamp_u_dict3['max_time_stamp3'].update({user_id:lagtime_mean3})
 if user_id in user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].keys() :
delta_prior_question_elapsed_time[i]=prior_question_elapsed_time-user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]=prior_question_elapsed_time
else:
delta_prior_question_elapsed_time[i]=delta_prior_question_elapsed_time_mean
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].update({user_id:prior_question_elapsed_time})
 i=i+1
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df=test_df.merge(questions_df.loc[questions_df.index.isin(test_df['content_id'])],
how='left', on='content_id', right_index=True)
 test_df['user_lecture_lv'] = user_lecture_sum / user_lecture_count
test_df['user_lecture_sum'] = user_lecture_sum
test_df['user_interaction_count'] = user_lecture_count
test_df['user_interaction_timestamp_mean'] = test_df['timestamp']/user_lecture_count
test_df['user_correctness'] = user_sum / user_count
test_df['user_uncorrect_count'] =user_count-user_sum
test_df['user_correct_count'] =user_sum
test_df['task_container_correctness'] = task_container_sum / task_container_count
test_df['task_container_cor_count'] = task_container_sum
test_df['task_container_uncor_count'] =task_container_count-task_container_sum
test_df['task_container_std'] = task_container_std
test_df['explanation_mean'] = explanation_sum / explanation_count
test_df['explanation_true_count'] = explanation_sum
test_df['explanation_false_count'] = explanation_count-explanation_sum
test_df['delta_prior_question_elapsed_time'] = delta_prior_question_elapsed_time
test_df["attempt_no"] = test_df[["user_id", "content_id"]].apply(lambda row: get_max_attempt(row["user_id"], row["content_id"]), axis=1)
 test_df["lagtime"]=lagtime
test_df["lagtime2"]=lagtime2
test_df["lagtime3"]=lagtime3
test_df['timestamp']=test_df['timestamp']/(1000*3600)
 test_df.timestamp=test_df.timestamp.astype('float16')
 test_df['lagtime']=test_df['lagtime']/(1000*3600)
 test_df.lagtime=test_df.lagtime.astype('float32')
 test_df['lagtime2']=test_df['lagtime2']/(1000*3600)
 test_df.lagtime2=test_df.lagtime2.astype('float32')
 test_df['lagtime3']=test_df['lagtime3']/(1000*3600)
 test_df.lagtime3=test_df.lagtime3.astype('float32')
 test_df['user_interaction_timestamp_mean']=test_df['user_interaction_timestamp_mean']/(1000*3600)
 test_df.user_interaction_timestamp_mean=test_df.user_interaction_timestamp_mean.astype('float32')
 test_df['user_correctness'].fillna(0.67, inplace=True)
 sub_preds = np.zeros(test_df.shape[0])
 for i, model in enumerate(clfs, 1):
test_preds = model.predict(test_df[features])
 sub_preds += test_preds
o2=sub_preds / len(clfs)
 test_df[target]=0.5*np.array(outs)+0.5*np.array(o2)
 env.predict(test_df[['row_id', target]] )