Kaggle competition: Tabular Playground Series - Feb 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-feb-2021<split>
import optuna
env = riiideducation.make_env()
iter_test = env.iter_test()
prior_test_df = None<data_type_conversions>
from lightgbm import LGBMRegressor
import numpy as np
import pandas as pd
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
%%time
for(test_df, sample_prediction_df)in iter_test:
if(prior_test_df is not None)&(psutil.virtual_memory().percent<90):
print(psutil.virtual_memory().percent)
 prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)
 prior_test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 prior_test_df.prior_question_had_explanation=prior_test_df.prior_question_had_explanation.astype('int8')
 prev_group = prior_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
prev_group_content = prev_group[prev_user_id][0]
prev_group_ac = prev_group[prev_user_id][1]
if prev_user_id in group.index:
group[prev_user_id] =(np.append(group[prev_user_id][0],prev_group_content),
np.append(group[prev_user_id][1],prev_group_ac))
else:
group[prev_user_id] =(prev_group_content,prev_group_ac)
 if len(group[prev_user_id][0])>MAX_SEQ:
new_group_content = group[prev_user_id][0][-MAX_SEQ:]
new_group_ac = group[prev_user_id][1][-MAX_SEQ:]
group[prev_user_id] =(new_group_content,new_group_ac)
 user_ids = prior_test_df['user_id'].values
content_ids = prior_test_df['content_id'].values
task_container_ids = prior_test_df['task_container_id'].values
prior_question_had_explanations = prior_test_df['prior_question_had_explanation'].values
targets = prior_test_df[target].values
for user_id, content_id,prior_question_had_explanation,task_container_id,answered_correctly in zip(user_ids, content_ids, prior_question_had_explanations,task_container_ids,targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
explanation_sum_dict[user_id] += prior_question_had_explanation
explanation_count_dict[user_id] += 1
prior_test_df = test_df.copy()
lecture_test_df = test_df[test_df['content_type_id'] == 1].reset_index(drop=True)
 for i,(user_id,content_type_id, content_id)in enumerate(zip(lecture_test_df['user_id'].values,lecture_test_df['content_type_id'].values,lecture_test_df['content_id'].values)) :
user_lecture_sum_dict[user_id] += content_type_id
user_lecture_count_dict[user_id] += 1
if(len(user_lecture_stats_part[user_lecture_stats_part.user_id==user_id])==0):
user_lecture_stats_part = user_lecture_stats_part.append([{'user_id':user_id}], ignore_index=True)
 user_lecture_stats_part.fillna(0, inplace=True)
 user_lecture_stats_part.loc[user_lecture_stats_part.user_id==user_id,part_lectures_columns + types_of_lectures_columns]+=lectures_df[lectures_df.lecture_id==content_id][part_lectures_columns + types_of_lectures_columns].values
else:
user_lecture_stats_part.loc[user_lecture_stats_part.user_id==user_id,part_lectures_columns + types_of_lectures_columns]+=lectures_df[lectures_df.lecture_id==content_id][part_lectures_columns + types_of_lectures_columns].values
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_dataset = TestDataset(group, test_df, skills)
 test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)
 SAKT_outs = []
for item in test_dataloader:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = SAKT_model(x, target_id)
 output = torch.sigmoid(output)
 output = output[:, -1]
SAKT_outs.extend(output.view(-1 ).data.cpu().numpy())
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df.prior_question_had_explanation=test_df.prior_question_had_explanation.astype('int8')
 test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)
 user_lecture_sum = np.zeros(len(test_df), dtype=np.int16)
 user_lecture_count = np.zeros(len(test_df), dtype=np.int16)
 user_sum = np.zeros(len(test_df), dtype=np.int16)
 user_count = np.zeros(len(test_df), dtype=np.int16)
 content_sum = np.zeros(len(test_df), dtype=np.int32)
 content_count = np.zeros(len(test_df), dtype=np.int32)
 task_container_sum = np.zeros(len(test_df), dtype=np.int32)
 task_container_count = np.zeros(len(test_df), dtype=np.int32)
 task_container_std = np.zeros(len(test_df), dtype=np.float16)
 content_task_mean = np.zeros(len(test_df), dtype=np.float16)
 explanation_sum = np.zeros(len(test_df), dtype=np.int32)
 explanation_count = np.zeros(len(test_df), dtype=np.int32)
 delta_prior_question_elapsed_time = np.zeros(len(test_df), dtype=np.int32)
 attempt_no_count = np.zeros(len(test_df), dtype=np.int16)
 lagtime = np.zeros(len(test_df), dtype=np.int32)
 lagtime_mean = np.zeros(len(test_df), dtype=np.int32)
 for i,(user_id,prior_question_had_explanation,content_type_id,prior_question_elapsed_time,timestamp, content_id,task_container_id)in enumerate(zip(test_df['user_id'].values,test_df['prior_question_had_explanation'].values,test_df['content_type_id'].values,test_df['prior_question_elapsed_time'].values,test_df['timestamp'].values, test_df['content_id'].values, test_df['task_container_id'].values)) :
user_lecture_sum_dict[user_id] += content_type_id
user_lecture_count_dict[user_id] += 1
user_lecture_sum[i] = user_lecture_sum_dict[user_id]
user_lecture_count[i] = user_lecture_count_dict[user_id]
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
content_sum[i] = content_sum_dict[content_id]
content_count[i] = content_count_dict[content_id]
task_container_sum[i] = task_container_sum_dict[task_container_id]
task_container_count[i] = task_container_count_dict[task_container_id]
task_container_std[i]=task_container_std_dict[task_container_id]
explanation_sum[i] = explanation_sum_dict[user_id]
explanation_count[i] = explanation_count_dict[user_id]
if user_id in max_timestamp_u_dict['max_time_stamp'].keys() :
lagtime[i]=timestamp-max_timestamp_u_dict['max_time_stamp'][user_id]
max_timestamp_u_dict['max_time_stamp'][user_id]=timestamp
lagtime_mean[i]=(lagtime_mean_dict[user_id]+lagtime[i])/2
else:
lagtime[i]=0
max_timestamp_u_dict['max_time_stamp'].update({user_id:timestamp})
 lagtime_mean_dict.update({user_id:timestamp})
 lagtime_mean[i]=(lagtime_mean_dict[user_id]+lagtime[i])/2
if user_id in user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].keys() :
delta_prior_question_elapsed_time[i]=prior_question_elapsed_time-user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]=prior_question_elapsed_time
else:
delta_prior_question_elapsed_time[i]=0
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].update({user_id:prior_question_elapsed_time})
 test_df=test_df.merge(questions_df.loc[questions_df.index.isin(test_df['content_id'])],
how='left', on='content_id', right_index=True)
 test_df['user_lecture_lv'] = user_lecture_sum / user_lecture_count
test_df['user_lecture_cumsum'] = user_lecture_sum
test_df['user_correctness'] = user_sum / user_count
test_df['user_correct_cumcount'] =user_count
test_df['user_correct_cumsum'] =user_sum
test_df['content_correctness'] = content_sum / content_count
test_df['content_count'] = content_count
test_df['content_sum'] = content_sum
test_df['task_container_correctness'] = task_container_sum / task_container_count
test_df['task_container_sum'] = task_container_sum
test_df['task_container_std'] = task_container_std
test_df['explanation_mean'] = explanation_sum / explanation_count
test_df['explanation_cumsum'] = explanation_sum
test_df['delta_prior_question_elapsed_time'] = delta_prior_question_elapsed_time
test_df["attempt_no"] = test_df[["user_id", "content_id"]].apply(lambda row: get_max_attempt(row["user_id"], row["content_id"]), axis=1)
 test_df["lagtime"]=lagtime
test_df["lagtime_mean"]=lagtime_mean
test_df['user_correctness'].fillna(1, inplace=True)
 test_df['attempt_no'].fillna(1, inplace=True)
 test_df.fillna(0, inplace=True)
 test_df['timestamp']=test_df['timestamp']/(1000*3600)
 test_df.timestamp=test_df.timestamp.astype('int16')
 sub_preds = np.zeros(test_df.shape[0])
 for i, model in enumerate(clfs, 1):
test_preds = model.predict(test_df[features], num_iteration = model.best_iteration)
 sub_preds += test_preds
lgbm_final = sub_preds / len(clfs)
 test_df[target] = np.array(SAKT_outs)* 0.5 + lgbm_final * 0.5
env.predict(test_df[['row_id', target]] )<load_pretrained>
train = pd.read_csv('.. /input/tabular-playground-series-feb-2021/train.csv')
 test = pd.read_csv('.. /input/tabular-playground-series-feb-2021/test.csv')
 sub = pd.read_csv('.. /input/tabular-playground-series-feb-2021/sample_submission.csv' )
pickle.dump(test_df, open('test_df.pickle', 'wb'))<save_model>
categorical_cols=['cat'+str(i)for i in range(10)]
continous_cols=['cont'+str(i)for i in range(14)]
model.save_model('model.txt')
 print('Save the best of model.' )<load_pretrained>
for e in categorical_cols:
le = LabelEncoder()
train[e]=le.fit_transform(train[e])
 test[e]=le.transform(test[e] )
pickle.dump(params, open('The_best_of_all_parameters_on_LightGBM.pickle', 'wb'))
print('Save the best of parameters.' )<load_from_csv>
data=train[categorical_cols+continous_cols]
target=train['target']
target = 'answered_correctly'
y_true = pd.read_csv('.. /input/riiid-test-answer-prediction/train.csv', usecols=[target])
 print('Length of y_true =', len(y_true))
y_pred = pd.read_csv('./submission.csv', usecols=[target])
 print('Length of y_pred =', len(y_pred))
test_df = pd.read_pickle('./test_df.pickle')
 print('Length of test_df =', len(test_df))<load_pretrained>
def objective(trial,data=data,target=target):
train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.2,random_state=42)
 param = {
'metric': 'rmse',
'random_state': 48,
'n_estimators': 20000,
'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-3, 10.0),
'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-3, 10.0),
'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.3,0.4,0.5,0.6,0.7,0.8,0.9, 1.0]),
'subsample': trial.suggest_categorical('subsample', [0.4,0.5,0.6,0.7,0.8,1.0]),
'learning_rate': trial.suggest_categorical('learning_rate', [0.006,0.008,0.01,0.014,0.017,0.02]),
'max_depth': trial.suggest_categorical('max_depth', [10,20,100]),
'num_leaves' : trial.suggest_int('num_leaves', 1, 1000),
'min_child_samples': trial.suggest_int('min_child_samples', 1, 300),
'cat_smooth' : trial.suggest_int('min_data_per_groups', 1, 100)
 }
model = LGBMRegressor(**param)
 model.fit(train_x,train_y,eval_set=[(test_x,test_y)],early_stopping_rounds=100,verbose=False)
 preds = model.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 return rmse
params = pd.read_pickle('./The_best_of_all_parameters_on_LightGBM.pickle')
 print('Optimized parameters with Optuna:')
 print(params )<choose_model_class>
study = optuna.create_study(direction='minimize')
 study.optimize(objective, n_trials=50)
 print('Number of finished trials:', len(study.trials))
print('Best trial:', study.best_trial.params )
model = lgb.Booster(model_file='./model.txt', params=params)
 print('Load the model of LightGBM:')
 print(model )<predict_on_test>
study.trials_dataframe()
LightGBM = model.predict(test_df[features])
 LightGBM<normalization>
params=study.best_params
params['random_state'] = 48
params['n_estimators'] = 20000
params['metric'] = 'rmse'
y_true_boot = resample(y_true, n_samples = len(y_pred))<compute_test_metric>
params['cat_smooth'] = params.pop('min_data_per_groups' )
def Statistical_Significance_Test(y_true_boot, y_pred):
A = np.array(y_true_boot)
 B = np.array(y_pred)
 print('Paired t-test:', stats.ttest_rel(A, B))
A_var = np.var(A, ddof=1)
 B_var = np.var(B, ddof=1)
 A_df = len(A)- 1
B_df = len(B)- 1
f = A_var / B_var
one_sided_pval1 = stats.f.cdf(f, A_df, B_df)
 one_sided_pval2 = stats.f.sf(f, A_df, B_df)
 two_sided_pval = min(one_sided_pval1, one_sided_pval2)* 2
print("Student's t-test:")
 print('F: ', round(f, 3))
print('p-value: ', round(two_sided_pval, 3))
two_sided_pval = min(one_sided_pval1, one_sided_pval2)* 2
print("Welch's t-test:")
 print('F: ', round(f, 3))
print('p-value: ', round(two_sided_pval, 4))
print("Mann-Whitney's U-test:", stats.mannwhitneyu(A, B, alternative='two-sided'))
print("Wilcoxon signed rank sum test:", stats.wilcoxon(A, B))<compute_test_metric>
columns = categorical_cols+continous_cols
preds = np.zeros(test.shape[0])
 kf = KFold(n_splits=5,random_state=48,shuffle=True)
 rmse=[]
n=0
for trn_idx, test_idx in kf.split(train[columns],train['target']):
X_tr,X_val=train[columns].iloc[trn_idx],train[columns].iloc[test_idx]
y_tr,y_val=train['target'].iloc[trn_idx],train['target'].iloc[test_idx]
model = LGBMRegressor(**params)
 model.fit(X_tr,y_tr,eval_set=[(X_val,y_val)],early_stopping_rounds=100,verbose=False)
 preds+=model.predict(test[columns])/kf.n_splits
rmse.append(mean_squared_error(y_val, model.predict(X_val), squared=False))
print(n+1,rmse[n])
 n+=1
<compute_test_metric><EOS> 
sub['target']=preds
sub.to_csv('submission.csv', index=False )