Kaggle competition: Tabular Playground Series - Feb 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-feb-2021<data_type_conversions>
%%html
<style>
table { float:left; margin-bottom: 1em; }
table + * { content: ""; clear: both; display: table; }
</style>
train_df['content_count'] = train_df['content_id'].map(content_agg['count'] ).astype('int32')
 train_df['content_sum'] = train_df['content_id'].map(content_agg['sum'] ).astype('int32')
 train_df['content_correctness'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 train_df.content_correctness=train_df.content_correctness.astype('float16')
 train_df['task_container_sum'] = train_df['task_container_id'].map(task_container_agg['sum'] ).astype('int32')
 train_df['task_container_std'] = train_df['task_container_id'].map(task_container_agg['var'] ).astype('float16')
 train_df['task_container_correctness'] = train_df['task_container_id'].map(task_container_agg['sum'] / task_container_agg['count'])
 train_df.task_container_correctness=train_df.task_container_correctness.astype('float16' )<load_from_csv>
!pip3 install -q pycaret[full] 2> /dev/null
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 1,3,4],
dtype={'question_id': 'int16','bundle_id': 'int16', 'part': 'int8','tags': 'str'} 
)
 questions_df['part_bundle_id']=questions_df['part']*100000+questions_df['bundle_id']
questions_df.part_bundle_id=questions_df.part_bundle_id.astype('int32')
 tag = questions_df["tags"].str.split(" ", n = 10, expand = True)
 tag.columns = ['tags1','tags2','tags3','tags4','tags5','tags6']
tag.fillna(0, inplace=True)
 tag = tag.astype('int16')
 questions_df = pd.concat([questions_df,tag],axis=1 ).drop(['tags'],axis=1 )<rename_columns>
version()
questions_df.rename(columns={'question_id':'content_id'}, inplace=True )<data_type_conversions>
data = pd.read_csv('.. /input/tabular-playground-series-feb-2021/train.csv', index_col='id')
 data
questions_df['content_correctness'] = questions_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 questions_df.content_correctness=questions_df.content_correctness.astype('float16')
 questions_df['content_correctness_std'] = questions_df['content_id'].map(content_agg['var'])
 questions_df.content_correctness_std=questions_df.content_correctness_std.astype('float16' )<categorify>
%%time
reg1 = setup(data, target='target', session_id=42, log_experiment=False, experiment_name='tabular-playground-feb-2021', silent=True, use_gpu=True )
part_agg = questions_df.groupby('part')['content_correctness'].agg(['mean', 'var'])
 questions_df['part_correctness_mean'] = questions_df['part'].map(part_agg['mean'])
 questions_df['part_correctness_std'] = questions_df['part'].map(part_agg['var'])
 questions_df.part_correctness_mean=questions_df.part_correctness_mean.astype('float16')
 questions_df.part_correctness_std=questions_df.part_correctness_std.astype('float16' )<categorify>
%%time
lightgbm = create_model('lightgbm',
bagging_fraction=0.7, bagging_freq=7, boosting_type='gbdt',
class_weight=None, colsample_bytree=1.0, device='gpu',
feature_fraction=0.5, importance_type='split',
learning_rate=0.071, max_depth=-1, min_child_samples=70,
min_child_weight=0.001, min_split_gain=0.8, n_estimators=150,
n_jobs=-1, num_leaves=100, objective=None, random_state=42,
reg_alpha=5, reg_lambda=0.15, silent=True, subsample=1.0,
subsample_for_bin=200000, subsample_freq=0
)
bundle_agg = questions_df.groupby('bundle_id')['content_correctness'].agg(['mean'])
 questions_df['bundle_correctness'] = questions_df['bundle_id'].map(bundle_agg['mean'])
 questions_df.bundle_correctness=questions_df.bundle_correctness.astype('float16' )<categorify>
%%time
catboost = create_model('catboost',
depth = 8,
l2_leaf_reg = 4,
loss_function = 'RMSE',
border_count = 32,
random_strength = 0.7,
task_type = 'GPU',
n_estimators = 290,
)
tags1_agg = questions_df.groupby('tags1')['content_correctness'].agg(['mean', 'var'])
 questions_df['tags1_correctness_mean'] = questions_df['tags1'].map(tags1_agg['mean'])
 questions_df['tags1_correctness_std'] = questions_df['tags1'].map(tags1_agg['var'])
 questions_df.tags1_correctness_mean=questions_df.tags1_correctness_mean.astype('float16')
 questions_df.tags1_correctness_std=questions_df.tags1_correctness_std.astype('float16' )<drop_column>
%%time
xgboost = create_model('xgboost',
base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.5, gamma=0, gpu_id=0,
importance_type='gain', interaction_constraints='',
learning_rate=0.274, max_delta_step=0, max_depth=3,
min_child_weight=3, monotone_constraints='() ',
n_estimators=200, n_jobs=-1, num_parallel_tree=1,
objective='reg:squarederror', reg_alpha=0.7,
reg_lambda=0.15, scale_pos_weight=48.30000000000001, subsample=0.7,
tree_method='gpu_hist', validate_parameters=1, verbosity=0
)
questions_df.drop(columns=['content_correctness'], inplace=True )<drop_column>
%%time
bayesian_ridge = create_model('br',
alpha_1=0.05, alpha_2=0.0005, alpha_init=None,
compute_score=False, copy_X=True, fit_intercept=True,
lambda_1=0.2, lambda_2=0.0005, lambda_init=None, n_iter=300,
normalize=False, tol=0.001, verbose=False
)
del bundle_agg
del part_agg
del tags1_agg
gc.collect()<train_model>
%%time
dt = create_model('dt')
 bagged_dt = ensemble_model(dt, n_estimators=10, optimize='RMSE' )
train_df['user_correctness'].fillna(1, inplace=True)
 train_df['attempt_no'].fillna(1, inplace=True)
 train_df.fillna(0, inplace=True )<categorify>
%%time
bagged_lightgbm = ensemble_model(lightgbm, method='Bagging', n_estimators=10, optimize='RMSE' )
MAX_SEQ = 160
skills = train_df["content_id"].unique()
n_skill = len(skills)
 print("number skills", len(skills))
group = train_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for user_id in group.index:
q, qa = group[user_id]
if len(q)>MAX_SEQ:
group[user_id] =(q[-MAX_SEQ:],qa[-MAX_SEQ:])
 pickle.dump(group, open("group.pkl", "wb"))
del group
gc.collect() 
<define_variables>
%%time
bagged_catboost = ensemble_model(catboost, method='Bagging', n_estimators=10, optimize='RMSE' )
features = [
'lagtime',
'lagtime_mean',
'user_lecture_cumsum',
'user_lecture_lv',
'prior_question_elapsed_time',
'delta_prior_question_elapsed_time',
'user_correctness',
'user_correct_cumcount',
'user_correct_cumsum',
'content_correctness',
'content_count',
'content_sum',
'task_container_correctness',
'bundle_correctness',
'attempt_no',
'part',
'part_correctness_mean',
'tags1',
'tags1_correctness_mean',
'bundle_id',
'explanation_mean',
'explanation_cumsum',
'prior_question_had_explanation',
]
categorical_columns= [
'part',
'tags1',
'bundle_id',
'prior_question_had_explanation',
]<define_variables>
%%time
bagged_xgboost = ensemble_model(xgboost, method='Bagging', n_estimators=10, optimize='RMSE' )
flag_lgbm=True
clfs = list()<load_pretrained>
%%time
blender = blend_models(optimize='RMSE', estimator_list=[
bayesian_ridge,
lightgbm,
catboost,
xgboost,
bagged_lightgbm,
bagged_catboost,
bagged_xgboost,
bagged_dt,
] )
params = pd.read_pickle('.. /input/the-best-of-all-parameters-on-lightgbm/The_best_of_all_parameters_on_LightGBM.pickle')
 print('Optimized parameters with Optuna:')
 print(params )<merge>
%%time
best = automl(optimize='RMSE')
 best
trains=list()
valids=list()
num=1
for i in range(0,num):
train_df_clf=train_df.sample(n=20000*1000, random_state = SEED)
 print('sample end')
 del train_df
users=train_df_clf['user_id'].drop_duplicates()
users=users.sample(frac=0.025, random_state = SEED)
 users_df=pd.DataFrame()
users_df['user_id']=users.values
valid_df_newuser = pd.merge(train_df_clf, users_df, on=['user_id'], how='inner',right_index=True)
 del users_df
del users
gc.collect()
train_df_clf.drop(valid_df_newuser.index, inplace=True)
 train_df_clf = pd.merge(train_df_clf, questions_df, on='content_id', how='left',right_index=True)
 valid_df_newuser = pd.merge(valid_df_newuser, questions_df, on='content_id', how='left',right_index=True)
 valid_df=train_df_clf.sample(frac=0.09, random_state = SEED)
 train_df_clf.drop(valid_df.index, inplace=True)
 valid_df = valid_df.append(valid_df_newuser)
 del valid_df_newuser
gc.collect()
trains.append(train_df_clf)
 valids.append(valid_df)
 print('valid_df length：',len(valid_df))<set_options>
%%time
evaluate_model(best )
del train_df_clf
del valid_df
gc.collect()<prepare_x_and_y>
%%time
pred_holdouts = predict_model(best)
 pred_holdouts.head()
for i in range(0, num):
tr_data = lgb.Dataset(trains[i][features], label=trains[i][target])
 va_data = lgb.Dataset(valids[i][features], label=valids[i][target], reference = tr_data)
 del trains
del valids
gc.collect()
model = lgb.train(
 params,
tr_data,
num_boost_round=10000,
valid_sets=[tr_data, va_data],
early_stopping_rounds=50,
feature_name=features,
categorical_feature=categorical_columns,
verbose_eval=50 
)
 clfs.append(model)
 del tr_data
del va_data
gc.collect() 
<data_type_conversions>
%%time
new_data = pd.read_csv('.. /input/tabular-playground-series-feb-2021/test.csv', index_col='id')
 predict_new = predict_model(best, data=new_data)
 predict_new.head()
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int))
content_sum_dict = content_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
content_count_dict = content_agg['count'].astype('int32' ).to_dict(defaultdict(int))
del user_agg
del content_agg
gc.collect()
task_container_sum_dict = task_container_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
task_container_count_dict = task_container_agg['count'].astype('int32' ).to_dict(defaultdict(int))
task_container_std_dict = task_container_agg['var'].astype('float16' ).to_dict(defaultdict(int))
explanation_sum_dict = explanation_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
explanation_count_dict = explanation_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del task_container_agg
del explanation_agg
gc.collect()<data_type_conversions>
submission_df = pd.read_csv('.. /input/tabular-playground-series-feb-2021/sample_submission.csv', index_col='id')
 submission_df['target'] = predict_new['Label']
submission_df.to_csv('submission.csv')
 !head submission.csv
user_lecture_sum_dict = user_lecture_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_lecture_count_dict = user_lecture_agg['count'].astype('int16' ).to_dict(defaultdict(int))
lagtime_mean_dict = lagtime_agg['mean'].astype('int32' ).to_dict(defaultdict(int))
del user_lecture_agg
del lagtime_agg
gc.collect()<categorify>
save_model(best, model_name='best-model' )
attempt_no_agg=attempt_no_agg[attempt_no_agg['sum'] >1]
attempt_no_sum_dict = attempt_no_agg['sum'].to_dict(defaultdict(int))
del attempt_no_agg
gc.collect()<categorify>
loaded_bestmodel = load_model('best-model')
 print(loaded_bestmodel )
max_timestamp_u_dict=max_timestamp_u.set_index('user_id' ).to_dict()
user_prior_question_elapsed_time_dict=user_prior_question_elapsed_time.set_index('user_id' ).to_dict()
del max_timestamp_u
del user_prior_question_elapsed_time
gc.collect()<feature_engineering>
set_config(display='diagram')
 loaded_bestmodel[0]
def get_max_attempt(user_id,content_id):
k =(user_id,content_id)
 if k in attempt_no_sum_dict.keys() :
attempt_no_sum_dict[k]+=1
return attempt_no_sum_dict[k]
attempt_no_sum_dict[k] = 1
return attempt_no_sum_dict[k]<choose_model_class>
set_config(display='text' )
<load_pretrained><EOS>