Kaggle competition: Riiid Answer Correctness Prediction
y_pred = pd.DataFrame(XGB.predict(df_test))
y_pred['Survived'] = y_pred[0]
y_pred.drop(0,axis=1,inplace=True)
 y_pred['PassengerId'] = df_test['PassengerId']
y_pred_xgb = y_pred
y_pred.to_csv('titanic_pred_xgb.csv',index=False )<normalization>
full_train = pd.read_pickle(".. /input/riiid-train-data-multiple-formats/riiid_train.pkl.gzip")
 questions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')
 lectures = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )
Scaler1 = StandardScaler()
Scaler2 = StandardScaler()
X_train_scaled = Scaler1.fit_transform(X_train)
 df_test_scaled = Scaler2.fit_transform(df_test )<save_to_csv>
logmodel = LogisticRegression(C=10 ).fit(X_train,y_train)
 y_pred = pd.DataFrame(logmodel.predict(df_test))
y_pred['Survived'] = y_pred[0]
y_pred.drop(0,axis=1,inplace=True)
 y_pred['PassengerId'] = df_test['PassengerId']
y_pred_lr = y_pred
y_pred.to_csv('titanic_pred_logistic.csv',index=False )<train_model>
RFC = RandomForestClassifier(n_estimators=500,max_depth=9,min_samples_split=3)
 RFC.fit(X_train,y_train )<save_to_csv>
y_pred = pd.DataFrame(RFC.predict(df_test))
y_pred['Survived'] = y_pred[0]
y_pred.drop(0,axis=1,inplace=True)
 y_pred['PassengerId'] = df_test['PassengerId']
y_pred_rf = y_pred
y_pred.to_csv('titanic_pred_rfc.csv',index=False )<train_model>
lgb = LGBMClassifier(learning_rate=0.01,max_depth=5,n_estimators=500,num_leaves=3 ).fit(X_train,y_train )<save_to_csv>
y_pred = pd.DataFrame(lgb.predict(df_test))
y_pred['Survived'] = y_pred[0]
y_pred.drop(0,axis=1,inplace=True)
 y_pred['PassengerId'] = df_test['PassengerId']
y_pred_lgb = y_pred
y_pred.to_csv('titanic_pred_lgb.csv',index=False )<compute_test_metric>
print("XGB train score: ",round(XGB.score(X_train,y_train),2), " XGB test score: ",round(XGB.score(X_test,y_test),2))
print("Log-Reg.train score: ",round(logmodel.score(X_train,y_train),2)," Log-Reg.test score: ",round(logmodel.score(X_test,y_test),2))
print("Random Forest's train score: ",round(RFC.score(X_train,y_train),2), " Random Forest test score: ",round(RFC.score(X_test,y_test),2))
print("LGBM train score: ",round(lgb.score(X_train,y_train),2), " LGB Model test score: ",round(lgb.score(X_test,y_test),2))<predict_on_test>
y_valid_xgb = XGB.predict(X_test)
 y_valid_log = logmodel.predict(X_test)
 y_valid_rfc = RFC.predict(X_test)
 y_valid_lgb = lgb.predict(X_test )<compute_train_metric>
fpr_xgb, tpr_xgb, thresholds_xgb = roc_curve(y_test, y_valid_xgb)
 roc_auc_xgb = auc(fpr_xgb, tpr_xgb)
 fpr_log, tpr_log, thresholds_log = roc_curve(y_test, y_valid_log)
 roc_auc_log = auc(fpr_log, tpr_log)
 fpr_rfc, tpr_rfc, thresholds_rfc = roc_curve(y_test, y_valid_rfc)
 roc_auc_rfc = auc(fpr_rfc, tpr_rfc)
 fpr_lgb, tpr_lgb, thresholds_lgb = roc_curve(y_test, y_valid_lgb)
 roc_auc_lgb = auc(fpr_lgb, tpr_lgb )<save_to_csv>
full_train = full_train[['row_id','user_id','content_id','content_type_id','answered_correctly']]
train = full_train.groupby('user_id' ).tail(500)
 test = full_train.groupby('user_id' ).tail(4)
 train = train.drop(test.index )
y_pred_final = y_pred
y_pred_final['Survived'] = round(0.25 * y_pred_lgb['Survived']
+ 0.25 * y_pred_rf['Survived']
+ 0.25 * y_pred_xgb['Survived']
+ 0.25 * y_pred_lr['Survived'])
 y_pred_final['PassengerId'] = df_test['PassengerId']
y_pred_final['Survived'] = y_pred_final['Survived'].astype(int)
 y_pred_final.to_csv('titanic_pred_final.csv',index=False )<count_values>
question_average = pd.DataFrame(full_train.loc[full_train['content_type_id'] == 0].groupby(['content_id'])['answered_correctly'].mean() ).rename(columns={'answered_correctly':'question_average'})
 question_count = pd.DataFrame(full_train.loc[full_train['content_type_id'] == 0].groupby(['content_id'] ).size() ,columns=['question_count'])
 question_df = question_average.join(question_count)
 question_df['question_sum'] = question_df['question_average'] * question_df['question_count']
question_df = question_df.join(questions,how='outer')[['question_average','question_count','question_sum']]
question_df['question_average'].fillna(0,inplace=True)
 question_df['question_count'].fillna(0,inplace=True)
 question_df['question_sum'].fillna(0,inplace=True)
 del question_average,question_count
gc.collect()
y_pred_final['Survived'].value_counts()<set_options>
del full_train
gc.collect()
def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn 
<load_from_csv>
train = train.join(question_df,on=['content_id'], rsuffix='_question' )
X_train = pd.read_csv('.. /input/train.csv')
 X_test = pd.read_csv('.. /input/test.csv' )<drop_column>
mean_user = train.loc[train['content_type_id'] == False].groupby(['user_id'])['answered_correctly'].mean().mean()
mean_question = train.loc[train['content_type_id'] == False].groupby(['content_id'])['answered_correctly'].mean().mean()
X_train.drop(X_train[(X_train['GrLivArea']>4000)&(X_train['SalePrice']<300000)].index, inplace=True )<prepare_x_and_y>
train['user_shift_question'] = train.loc[train['content_type_id'] == False].groupby(['user_id'])['question_average'].shift()
cumulated_question = train.loc[train['content_type_id'] == False].groupby(['user_id'])['user_shift_question'].agg(['cumsum','cumcount'])
 train.loc[train['content_type_id'] == False,'average_past_questions'] = cumulated_question['cumsum'] / cumulated_question['cumcount']
train['average_past_questions'].fillna(mean_question,inplace=True)
 train.drop(['user_shift_question','question_count','question_sum'],axis=1,inplace=True)
 del cumulated_question
y = X_train['SalePrice']
test_id = X_test['Id']
X_train.drop('SalePrice', 1, inplace=True)
 X_train.drop("Id", axis = 1, inplace = True)
 X_test.drop("Id", axis = 1, inplace = True)
 X_train_idx = X_train.shape[0]
X_test_idx = X_test.shape[0]
data = pd.concat(( X_train,X_test)).reset_index(drop=True)
 print('Train index: %s
Test index: %s
Shape of our whole data(rows,columns): %s'%(X_train_idx, X_test_idx, data.shape))<sort_values>
train['user_shift'] = train.loc[train['content_type_id'] == False].groupby(['user_id'])['answered_correctly'].shift()
cumulated = train.loc[train['content_type_id'] == False].groupby(['user_id'])['user_shift'].agg(['cumsum', 'cumcount'])
 train.loc[train['content_type_id'] == False,'answered_correctly_user_average'] = cumulated['cumsum'] / cumulated['cumcount']
train['answered_correctly_user_average'].fillna(mean_user,inplace=True)
 train.drop(columns=['user_shift'], inplace=True)
 del cumulated
numeric_cols = []
non_numeric_cols = []
for key,val in data.isnull().sum().sort_values(ascending=False ).items() :
if val > 0:
print(key, val, data[key].dtype)
 if data[key].dtype != 'object':numeric_cols.append(key)
 elif data[key].dtype == 'object':non_numeric_cols.append(key)
<data_type_conversions>
user_average = pd.DataFrame(train.loc[train['content_type_id'] == 0].groupby(['user_id'])['answered_correctly_user_average'].last() ).rename(columns={'answered_correctly_user_average':'user_average'})
 user_count = pd.DataFrame(train.loc[train['content_type_id'] == 0].groupby(['user_id'] ).size() - 1,columns=['user_count'] )
fill_zero_cols = ['BsmtHalfBath', 'BsmtFullBath', 'BsmtFinSF2', 'GarageCars']
fill_median_cols = ['GarageArea','TotalBsmtSF', 'MasVnrArea', 'BsmtFinSF1', 'LotFrontage', 'BsmtUnfSF', 'GarageYrBlt']
[data[i].fillna(0,inplace=True)for i in fill_zero_cols]
[data[j].fillna(data[j].median() ,inplace=True)for j in fill_median_cols];<correct_missing_values>
tmp = train.loc[train['content_type_id'] == False].groupby(['user_id'] ).mean()
user_performance = pd.DataFrame(tmp['answered_correctly'] - tmp['question_average'], columns=['performance'])
 del tmp
data.fillna('none', inplace=True )<count_missing_values>
user_df = user_performance.join(user_average ).join(user_count)
 user_df['user_sum'] = user_df['user_average'] * user_df['user_count']
del user_performance, user_count, user_average
gc.collect()
data.isnull().sum().sort_values(ascending=False )<feature_engineering>
def question_average_sum_by_user(df,question_df):
my_dict = {}
group = df.groupby(['user_id'])
 for user, val in group:
average_sum = 0.0
for row_index, row in val.iterrows() :
if(row['content_type_id'] == False):
question_id = row['content_id']
question_average = question_df.at[question_id,'question_average']
average_sum += question_average
my_dict[user] = [average_sum]
return pd.DataFrame.from_dict(my_dict,orient='index',columns=['question_average_sum'] )
y = np.log1p(y)
 numeric_feats = data.dtypes[data.dtypes != "object"].index
skewed_feats = X_train[numeric_feats].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index
data[skewed_feats] = np.log1p(data[skewed_feats] )<categorify>
def add_answers_to_prior_df(current_df,prior_df):
prior_df_ = prior_df.copy()
if(prior_df.shape[0] > 0):
val = eval(current_df.iloc[0]['prior_group_answers_correct'])
 if(len(val)== prior_df.shape[0]):
prior_df_['answered_correctly_response'] = val
return prior_df_
data = pd.get_dummies(data )<split>
def build_question_df(prior_df,question_df):
if(prior_df.shape[0] == 0):
return question_df
question_sum_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['content_id'])['answered_correctly_response'].sum())\ 
.rename(columns={'answered_correctly_response':'question_sum'})
 question_count_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['content_id'] ).size() ,columns=['question_count'])
 question_df = question_df.join(question_sum_prior,rsuffix='_previous' ).join(question_count_prior,rsuffix='_previous')
 question_df['question_average'].fillna(0,inplace=True)
 question_df['question_count'].fillna(0,inplace=True)
 question_df['question_sum'].fillna(0,inplace=True)
 question_df['question_sum_previous'].fillna(0,inplace=True)
 question_df['question_count_previous'].fillna(0,inplace=True)
 question_df['question_sum'] = question_df['question_sum'] + question_df['question_sum_previous']
question_df['question_count'] = question_df['question_count'] + question_df['question_count_previous']
question_df['question_average'] = question_df['question_sum'] / question_df['question_count']
question_df.drop(['question_count_previous','question_sum_previous'],inplace=True,axis=1)
 return question_df
X_train = data[:X_train_idx]
X_test = data[X_train_idx:]<import_modules>
def build_user_df(prior_df,user_df,question_df):
if(prior_df.shape[0] == 0):
return user_df
user_sum_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['user_id'])['answered_correctly_response'].sum())\ 
.rename(columns={'answered_correctly_response':'user_sum'})
 user_count_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['user_id'] ).size() ,columns=['user_count'])
 user_df = user_df.join(user_sum_prior,how='outer',rsuffix='_previous' ).join(user_count_prior,rsuffix='_previous')
 user_df['performance'].fillna(0,inplace=True)
 user_df['user_average'].fillna(0,inplace=True)
 user_df['user_count'].fillna(0,inplace=True)
 user_df['user_sum'].fillna(0,inplace=True)
 user_df['user_count_previous'].fillna(0,inplace=True)
 user_df['user_sum_previous'].fillna(0,inplace=True)
 user_df = user_df.join(question_average_sum_by_user(prior_df,question_df))
user_df['question_average_sum'].fillna(0,inplace=True)
 user_df['user_mean_performance'] =(user_df['user_sum'] - user_df['performance'] * user_df['user_count'] + user_df['question_average_sum'])/(user_df['user_count'] + user_df['user_count_previous'])
 user_df['user_sum'] = user_df['user_sum'] + user_df['user_sum_previous']
user_df['user_count'] = user_df['user_count'] + user_df['user_count_previous']
user_df['user_average'] = user_df['user_sum'] / user_df['user_count']
user_df['performance'] = user_df['user_average'] - user_df['user_mean_performance']
user_df.drop(['user_sum_previous','user_count_previous','question_average_sum','user_mean_performance'],axis=1,inplace=True)
 return user_df
from sklearn.linear_model import Lasso, ElasticNet, Ridge, LinearRegression
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import PolynomialFeatures
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import StandardScaler<compute_train_metric>
prior_df = pd.DataFrame()
current_df = pd.DataFrame()
prior_df = add_answers_to_prior_df(current_df,prior_df)
 question_df = build_question_df(prior_df,question_df)
 user_df = build_user_df(prior_df,user_df,question_df )
elastic_net = ElasticNet(alpha=.5, l1_ratio=.5)
 scores = cross_val_score(elastic_net, X_train, y, scoring='neg_mean_squared_error', cv=5)
 rmse_scores = np.sqrt(-scores)
 display_scores(rmse_scores )<choose_model_class>
TARGET_COL = ['answered_correctly']
FEATURE_COLS = ['row_id', 'performance', 'question_average']
param_grid = [
{'alpha': [.0004,.0005], 'l1_ratio': [.5,.8, 1]},
]
grid_search = GridSearchCV(ElasticNet() , param_grid, cv=5,
scoring='neg_mean_squared_error',
return_train_score=True )<train_on_grid>
train = train[['row_id','user_id','content_id', 'content_type_id', 'answered_correctly']].reset_index()
grid_search.fit(X_train, y )<compute_train_metric>
def data_transform(df, is_training = True, is_validation = True):
df = df.join(question_df['question_average'],on=['content_id'],rsuffix='_question_average')
 df = df.join(user_df[['performance','user_average', 'user_count']],on=['user_id'],rsuffix='_right')
 df = df.loc[df['content_type_id'] == False]
if is_training or is_validation:
df = df[FEATURE_COLS + TARGET_COL]
else:
df = df[FEATURE_COLS]
return df
cvres = grid_search.cv_results_
for mean_score, params in zip(cvres["mean_test_score"], cvres["params"]):
print(np.sqrt(-mean_score), params )<train_on_grid>
%%time
train = data_transform(train )
lasso = Lasso(alpha=.1 ).fit(X_train,y)
 scores = cross_val_score(lasso,X_train,y,scoring='neg_mean_squared_error',cv=5)
 rmse_scores = np.sqrt(-scores)
 display_scores(rmse_scores )<train_on_grid>
%%time
test = data_transform(test,False,True )
lasso = Lasso(alpha=.0005 ).fit(X_train,y)
 scores = cross_val_score(lasso,X_train,y,scoring='neg_mean_squared_error',cv=5)
 rmse_scores = np.sqrt(-scores)
 display_scores(rmse_scores )<save_to_csv>
X_train = train[FEATURE_COLS]
y_train = train[TARGET_COL]
X_test = test[FEATURE_COLS]
y_test = test[TARGET_COL]
preds = lasso.predict(X_test)
 lasso_predictions = pd.DataFrame(np.exp(preds), test_id, columns=['SalePrice'] ).to_csv('lasso_predictions.csv' )<set_options>
params = {
'objective': 'binary',
'seed': 42,
'metric': 'auc',
'learning_rate': 0.05,
'max_bin': 1000,
'num_leaves': 80,
'num_iterations' : 100
}
lgb_train = lgb.Dataset(X_train.iloc[:,1:],y_train)
 lgb_val = lgb.Dataset(X_test.iloc[:,1:],y_test )
warnings.filterwarnings('ignore')
 get_ipython().magic(u'matplotlib inline')
 plt.style.use('ggplot')
 print("Modules imported
")
 print("Files in current directory:")
 print(check_output(["ls", ".. /input"] ).decode("utf8")) 
<load_from_csv>
del X_train,y_train
gc.collect()
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print("train ", train.shape)
 print("test ", test.shape )<count_values>
model = lgb.train(
 params,
lgb_train,
valid_sets=[lgb_train,lgb_val],
verbose_eval=1,
num_boost_round=100,
early_stopping_rounds=10
)
print(train["MSSubClass"].value_counts())
<concatenate>
%%time
predictions = pd.DataFrame(model.predict(X_test.iloc[:,1:]),index=X_test.index )
trainWprice = pd.DataFrame(train)
 trainNoPrice = trainWprice.drop("SalePrice", axis=1)
 full=pd.concat([trainNoPrice,test], ignore_index=True)
 full.drop(['Id'],axis=1, inplace=True)
 print("train ", train.shape)
 print("test ", test.shape)
 print("full without Id and no SalePrice ", full.shape )<train_model>
roc_auc_score(y_test,predictions[0] )
<categorify><EOS> 
%%time
env = riiideducation.make_env()
iter_test = env.iter_test()
iter_nb = 0
for(current_df, sample_prediction_df)in iter_test:
if(iter_nb != 0):
prior_df = add_answers_to_prior_df(current_df,prior_df)
 question_df = build_question_df(prior_df,question_df)
 user_df = build_user_df(prior_df,user_df,question_df)
 prior_df = current_df.copy()
current_df = data_transform(current_df,False,False)
 current_df['answered_correctly'] = model.predict(current_df.iloc[:,1:])
 iter_nb = 1
env.predict(current_df.loc[:, ['row_id', 'answered_correctly']] )