Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<categorify>
import numpy as np
import pandas as pd
import category_encoders as ce
import gc
from sklearn.preprocessing import LabelEncoder
from sklearn import preprocessing, metrics
big_train = pd.concat([train,pd.get_dummies(train['location'], prefix='loc')],axis=1)
 big_train['ConfirmedCases_neighbors']=np.log1p(big_train['ConfirmedCases_neighbors'])
 big_train.reset_index(inplace=True)
 big_train.drop(columns=["Id"],inplace=True )<feature_engineering>
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl
train = dt.fread(".. /input/riiid-test-answer-prediction/train.csv" ).to_pandas()
def df_add_deltas(df_old):
df=df_old.copy()
df=df.sort_values(by=['location', 'Date'])
 df['d_ConfirmedCases'] = df.groupby(['location'])['ConfirmedCases'].diff()
df['d_Fatalities'] = df.groupby(['location'])['Fatalities'].diff()
df.loc[df['d_Fatalities']<0,'d_Fatalities']=0
df.loc[df['d_ConfirmedCases']<0,'d_ConfirmedCases']=0
df['prev_ConfirmedCases']=df['ConfirmedCases']-df['d_ConfirmedCases']
df['prev_Fatalities']=df['Fatalities']-df['d_Fatalities']
df['prev_ConfirmedCases']=np.log1p(df['prev_ConfirmedCases'])
 df['prev_Fatalities']=np.log1p(df['prev_Fatalities'])
 first_day_stat=df[df['Date']=='2020-01-22']
df.drop(df[df['Date']=='2020-01-22'].index, inplace=True)
 return df,first_day_stat 
<feature_engineering>
big_train,first_day_stat=df_add_deltas(big_train )<drop_column>
%%time
data_types_dict = {
'row_id': 'int64',
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id': 'int8',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'boolean'
}
train = train.astype(data_types_dict)
 train['prior_question_had_explanation'].fillna(False, inplace=True )
big_train.reset_index(inplace=True,drop=True )<prepare_x_and_y>
gc.collect()
X=big_train.drop(columns=['Province_State','Country_Region','Date','ConfirmedCases','Fatalities','location',
'd_ConfirmedCases','d_Fatalities'])
 y=big_train['d_ConfirmedCases']
y_2=big_train['d_Fatalities']<define_variables>
train= train.drop(columns=['row_id','task_container_id','user_answer'] )
max_day=X['day_of_year'].max()
mask_train=X['day_of_year']<max_day-DT+1
mask_test=X['day_of_year']>=max_day-DT+1<prepare_x_and_y>
gc.collect()
X_train=X[mask_train]
X_test=X[mask_test]
y_train=y[mask_train]
y_test=y[mask_test]
y_train_2=y_2[mask_train]
y_test_2=y_2[mask_test]<count_unique_values>
%%time
train = train[train.content_type_id == 0]
train['attempt'] = train.assign(dif =(( train.content_id.diff() != 0)|(train.user_id.diff() != 0)) ).groupby(["user_id","content_id"] ).dif.cumsum()
train['attempt']=train['attempt'] - 1
train['attempt'] = train['attempt'].apply(lambda x: 4 if x >= 4 else x)
 train= train.drop(columns=['content_type_id'] )
X_test['day_of_year'].nunique()<drop_column>
gc.collect()
X_train.drop(columns=['day_of_year'],inplace=True)
 X_test.drop(columns=['day_of_year'],inplace=True)
 X_train.drop(columns=['day_of_week'],inplace=True)
 X_test.drop(columns=['day_of_week'],inplace=True)
 X.drop(columns=['day_of_year'],inplace=True)
 X.drop(columns=['day_of_week'],inplace=True)
<drop_column>
grouped_by_df = train[['user_id','answered_correctly']].groupby('user_id')
 train_user = grouped_by_df.agg({'answered_correctly': ['mean', 'sum','count']} ).copy()
train_user.columns=['user_mean_accuracy','user_count_correct','user_count']
X.drop(columns=['index'],inplace=True)
 X_train.drop(columns=['index'],inplace=True)
 X_test.drop(columns=['index'],inplace=True )<init_hyperparams>
grouped_by_df = train[['content_id','answered_correctly']].groupby('content_id')
 train_Q = grouped_by_df.agg({'answered_correctly': ['mean']} ).copy()
train_Q.columns=['Q_mean_accuracy']
if optimize_model:
model = xgb.XGBRegressor(random_state=42)
 n_estimators_grid = [500, 1000,1500]
max_depth_grid = [6, 8, 10]
param_grid = dict(max_depth=max_depth_grid, n_estimators=n_estimators_grid)
 grid_search = GridSearchCV(model, param_grid, scoring="neg_root_mean_squared_error", n_jobs=-1, cv=[(X[mask_train].index,X[mask_test].index)], verbose=1)
 grid_result = grid_search.fit(X,np.log1p(y))
print("Best: %f using %s" %(grid_result.best_score_, grid_result.best_params_))
print(grid_result.cv_results_ )<choose_model_class>
questions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv' )
if optimize_model:
model = xgb.XGBRegressor(random_state=42)
 n_estimators_grid = [400,500,600]
max_depth_grid = [5,6,7]
param_grid = dict(max_depth=max_depth_grid, n_estimators=n_estimators_grid)
 grid_search = GridSearchCV(model, param_grid, scoring="neg_root_mean_squared_error", n_jobs=-1, cv=[(X[mask_train].index,X[mask_test].index)], verbose=1)
 grid_result = grid_search.fit(X,np.log1p(y))
print("Best: %f using %s" %(grid_result.best_score_, grid_result.best_params_))
print(grid_result.cv_results_ )<choose_model_class>
tag = questions["tags"].str.split(" ", expand = True)
 tag.columns = ['tags1','tags2','tags3','tags4','tags5','tags6']
questions = pd.concat([questions,tag],axis=1)
 questions['tags1'] = pd.to_numeric(questions['tags1'], errors='coerce')
 questions['tags2'] = pd.to_numeric(questions['tags2'], errors='coerce')
 questions['tags3'] = pd.to_numeric(questions['tags3'], errors='coerce')
 questions['tags4'] = pd.to_numeric(questions['tags4'], errors='coerce')
 questions['tags5'] = pd.to_numeric(questions['tags5'], errors='coerce')
 questions['tags6'] = pd.to_numeric(questions['tags6'], errors='coerce')
 questions['tags'] = questions['tags'].astype(str)
 cat_features = ['tags']
encoder = LabelEncoder()
label_encoder = preprocessing.LabelEncoder()
for feature in cat_features:
encoded = label_encoder.fit_transform(questions[cat_features])
 questions[feature + '_labels'] = encoded
questions= questions.drop(columns=['tags1','tags2','tags3','tags4','tags5','tags6','bundle_id','correct_answer'])
 questions.fillna(0, inplace=True)
 questions
if optimize_model:
model = xgb.XGBRegressor(random_state=42)
 n_estimators_grid = [150,200,250]
max_depth_grid = [1,2,3]
param_grid = dict(max_depth=max_depth_grid, n_estimators=n_estimators_grid)
 grid_search = GridSearchCV(model, param_grid, scoring="neg_root_mean_squared_error", n_jobs=-1, cv=[(X[mask_train].index,X[mask_test].index)], verbose=1)
 grid_result = grid_search.fit(X,np.log1p(y))
print("Best: %f using %s" %(grid_result.best_score_, grid_result.best_params_))
print(grid_result.cv_results_ )<choose_model_class>
train_Q=train_Q.merge(
 questions,
how='left',
left_on='content_id',
right_on='question_id')
 train_Q.info()
reg = xgb.XGBRegressor(n_estimators=n_estimators,max_depth=max_depth,random_state=42)
 reg_2 = xgb.XGBRegressor(n_estimators=n_estimators,max_depth=max_depth,random_state=42 )<train_model>
questions = None
del(questions)
 gc.collect()
reg.fit(X_train,np.log1p(y_train))<predict_on_test>
def convertBoolean(x):
if str(x)== "False":
return 0
elif str(x)== "True":
return 1
else:
return 0
y_pred = reg.predict(X_test )<compute_test_metric>
def handleM(_dt_train, _train_user, _train_Q):
_dt_train['prior_question_had_explanation'].fillna(False, inplace=True)
 _dt_train["prior_question_had_explanation_enc"] = _dt_train['prior_question_had_explanation'].apply(convertBoolean)
 _dt_train = _dt_train.merge(_train_user, how = 'left', on = 'user_id')
 del _train_user
gc.collect()
_dt_train = _dt_train.merge(_train_Q, how = 'left', left_on = 'content_id',right_on = 'question_id')
 del _train_Q
gc.collect()
_dt_train['mean_user_content_accuracy'] =2 *(_dt_train['user_mean_accuracy'] * _dt_train['Q_mean_accuracy'])/(_dt_train['user_mean_accuracy'] + _dt_train['Q_mean_accuracy'])
 _dt_train = _dt_train.drop(columns=['user_id', 'content_id','prior_question_had_explanation','question_id'])
 _dt_train.fillna(0.5, inplace=True)
 return _dt_train
np.sqrt(mean_squared_error(y_pred,np.log1p(y_test)) )<compute_test_metric>
train = train.sort_values(['timestamp'], ascending=True)
 train_part_len=int(len(train)*0.66)
 train = train.iloc[train_part_len:,:]
X_train_2=X_train.copy()
X_train_2['d_confirmed']=y_train
X_test_2=X_test.copy()
X_test_2['d_confirmed']=y_pred<train_model>
gc.collect()
reg_2.fit(X_train_2,np.log1p(y_train_2))<predict_on_test>
dt_train=handleM(train, train_user, train_Q )
y_pred_2 = reg_2.predict(X_test_2 )<compute_test_metric>
train = None
del(train)
 gc.collect()
np.sqrt(mean_squared_error(y_pred_2,np.log1p(y_test_2)) )<feature_engineering>
features = [
'user_mean_accuracy',
'user_count_correct',
'user_count',
'Q_mean_accuracy',
'part',
'Q_mean_accuracy',
'attempt',
'prior_question_elapsed_time'
]
target = 'answered_correctly'
categorical_feature = ['part', 'tags', 'tags_label', 'prior_question_had_explanation']
categorical_feature_idxs = []
for v in categorical_feature:
try:
categorical_feature_idxs.append(features.index(v))
except:
pass
dt_y = dt_train[target]
dt_x = dt_train[features]
test=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/test.csv")
 test.rename(columns={'ForecastId':'Id'},inplace=True)
 test=add_location(test)
 test.set_index('location',inplace=True)
 test['day_of_year']=test['Date'].dt.dayofyear
test['day_of_week']=test['Date'].dt.dayofweek
test=add_days_passed(test,first_day)
 test=add_country_stat(test,country_stat )<drop_column>
dt_train = None
del(dt_train)
 gc.collect()
big_train=big_train.drop(columns=["index"])
<prepare_x_and_y>
valid_fraction = 0.05
valid_size = int(len(dt_x)* valid_fraction)
 train_x = dt_x[:-1 * valid_size]
valid_x = dt_x[-valid_size:]
train_y = dt_y[:-1 * valid_size]
valid_y = dt_y[-valid_size:]
if Make_submission==True:
results=[]
for d in days_to_predict:
print("Predicting {}".format(d))
if d in known:
print("Data Known")
 X=big_train.drop(columns=['Province_State','Country_Region','ConfirmedCases','Fatalities','location','Date',
'day_of_year','day_of_week','d_ConfirmedCases','d_Fatalities'])
 y=big_train['d_ConfirmedCases']
y_2=big_train['d_Fatalities']
mask_train=big_train['Date']<d
mask_val=big_train['Date']==d
X_train=X[mask_train]
y_train=y[mask_train]
y_train_2=y_2[mask_train]
X_val=X[mask_val]
y_val=y[mask_val]
y_val_2=y_2[mask_val]
reg = xgb.XGBRegressor(n_estimators=n_estimators,max_depth=max_depth,random_state=42)
 reg_2 = xgb.XGBRegressor(n_estimators=n_estimators,max_depth=max_depth,random_state=42)
 reg.fit(X_train,np.log1p(y_train))
y_pred = reg.predict(X_val)
 print("MSLE {}".format(mean_squared_error(y_pred,np.log1p(y_val))))
X_train_2=X_train.copy()
X_train_2['d_ConfirmedCases']=np.log1p(y_train)
 X_val_2=X_val.copy()
X_val_2['d_ConfirmedCases']=y_pred
reg_2.fit(X_train_2,np.log1p(y_train_2))
y_pred_2 = reg_2.predict(X_val_2)
 print("MSLE {}".format(mean_squared_error(y_pred_2,np.log1p(y_val_2))))
elif d-np.timedelta64(86400000000000,'ns')in known:
print("Data Known")
 X=big_train.drop(columns=['Province_State','Country_Region','ConfirmedCases','Fatalities','location','Date',
'day_of_year','day_of_week','d_ConfirmedCases','d_Fatalities'])
 y=big_train['d_ConfirmedCases']
y_2=big_train['d_Fatalities']
mask_train=big_train['Date']<d
X_train=X[mask_train]
y_train=y[mask_train]
y_train_2=y_2[mask_train]
reg = xgb.XGBRegressor(n_estimators=n_estimators,max_depth=max_depth,random_state=42)
 reg_2 = xgb.XGBRegressor(n_estimators=n_estimators,max_depth=max_depth,random_state=42)
 reg.fit(X_train,np.log1p(y_train))
X_train_2=X_train.copy()
X_train_2['d_ConfirmedCases']=np.log1p(y_train)
 reg_2.fit(X_train_2,np.log1p(y_train_2))
X_test=test[test['Date']==d]
day=X_test['day_of_year'].iloc[0]
country_info=big_train[big_train['day_of_year']==day-1].groupby(['Country_Region'])['ConfirmedCases'].sum()
border_cases=border_info.merge(country_info, how='inner',
left_on=['country_border_name'],right_on=['Country_Region'])
 border_cases=border_cases.groupby(['country_name'])['ConfirmedCases'].sum()
border_cases=border_cases.rename('ConfirmedCases_neighbors')
 X_test=X_test.merge(border_cases, how='left', left_on=['Country_Region'],right_on=['country_name'])
 X_test['ConfirmedCases_neighbors']=X_test['ConfirmedCases_neighbors'].fillna(0)
 X_test = pd.concat([X_test,pd.get_dummies(X_test['location'], prefix='loc')],axis=1)
 X_test['ConfirmedCases_neighbors']=np.log1p(X_test['ConfirmedCases_neighbors'])
 X_test=X_test.merge(big_train[big_train['day_of_year']==day-1][['location','ConfirmedCases','Fatalities']], how='left',
left_on=['location'],right_on=['location'])
 X_test.rename(columns={'ConfirmedCases':'prev_ConfirmedCases','Fatalities':'prev_Fatalities'},inplace=True)
 X_test['prev_ConfirmedCases']=np.log1p(X_test['prev_ConfirmedCases'])
 X_test['prev_Fatalities']=np.log1p(X_test['prev_Fatalities'])
 X_test.set_index('Id',inplace=True)
 y_test=reg.predict(X_test.drop(columns=['Province_State','Country_Region','location','Date','day_of_year','day_of_week']))
X_test['d_ConfirmedCases']=y_test
y_test=reg_2.predict(X_test.drop(columns=['Province_State','Country_Region','location','Date',
'day_of_year','day_of_week']))
X_test['d_Fatalities']=y_test
X_test['Id']=X_test.index
X_test=X_test.merge(big_train[big_train['day_of_year']==day-1][['location','ConfirmedCases','Fatalities']], how='left',
left_on=['location'],right_on=['location'])
 X_test.set_index('Id',inplace=True)
 print(X_test.head(5))
X_test['d_ConfirmedCases']=np.expm1(X_test['d_ConfirmedCases'])
 X_test['d_Fatalities']=np.expm1(X_test['d_Fatalities'])
 X_test['ConfirmedCases']+=X_test['d_ConfirmedCases']
X_test['Fatalities']+=X_test['d_Fatalities']
results.append(X_test[['ConfirmedCases','Fatalities']])
 if not d in known:
big_train=pd.concat([big_train,X_test])
<save_to_csv>
gc.collect()
if Make_submission==True:
submission=pd.read_csv(".. /input/covid19-global-forecasting-week-2/submission.csv")
 submission.drop(columns=['ConfirmedCases','Fatalities'],inplace=True)
 submission=submission.merge(pd.concat(results),left_on=['ForecastId'],right_index=True ).clip(lower=0)
 submission.to_csv('submission.csv',index=False )<load_from_csv>
dtrain = lgb.Dataset(train_x, label=train_y)
 dvalid = lgb.Dataset(valid_x, label=valid_y)
 params= {
'objective': 'binary',
'seed': 42,
'metric': 'auc',
'learning_rate': 0.05,
'max_bin': 1500,
'num_leaves': 70
}
model = lgb.train(
 params,
dtrain,
num_boost_round=2500,
valid_sets=[dtrain,dvalid],
early_stopping_rounds=20,
verbose_eval=50,
categorical_feature = categorical_feature_idxs,
feature_name = features, 
)
train_orig = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 train_orig.shape<define_variables>
env = riiideducation.make_env()
iter_test = env.iter_test()
ids_with_target_error = [328,443,513,2619,3640,3900,4342,5781,6552,6554,6570,6701,6702,6729,6861,7226]
train_orig[train_orig['id'].isin(ids_with_target_error)]<feature_engineering>
for(test_df, sample_prediction_df)in iter_test:
test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df["prior_question_had_explanation_enc"] = test_df['prior_question_had_explanation'].apply(convertBoolean)
 test_df = test_df[test_df.content_type_id == 0]
test_df['attempt'] = test_df.assign(dif =(( test_df.content_id.diff() != 0)|(test_df.user_id.diff() != 0)) ).groupby(["user_id","content_id"] ).dif.cumsum()
test_df['attempt']=test_df['attempt'] - 1
test_df['attempt'] = test_df['attempt'].apply(lambda x: 4 if x >= 4 else x)
 test_df = test_df.merge(train_user, how = 'left', on = 'user_id')
 test_df = test_df.merge(train_Q, how = 'left', left_on = 'content_id',right_on = 'question_id')
 test_df['mean_user_content_accuracy'] =2 *(test_df['user_mean_accuracy'] * test_df['Q_mean_accuracy'])/(test_df['user_mean_accuracy'] + test_df['Q_mean_accuracy'])
 test_df.fillna(0.5, inplace=True)
 test_df['answered_correctly'] = model.predict(test_df[features])
 env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )