Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<split>
!rm -r /opt/conda/lib/python3.6/site-packages/lightgbm
!git clone --recursive https://github.com/Microsoft/LightGBM
iter_test = env.iter_test()<merge>
!apt-get install -y -qq libboost-all-dev
for(test_df, sample_prediction_df)in iter_test:
test_df['task_container_id'] = test_df.task_container_id.mask(test_df.task_container_id > 9999, 9999)
 test_df = pd.merge(test_df, group3, left_on=['task_container_id'], right_index= True, how="left")
 test_df = pd.merge(test_df, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df = pd.merge(test_df, results_u_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, results_u2_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, user_lecture_stats_part, on=['user_id'], how="left")
 test_df['part_1'].fillna(0, inplace = True)
 test_df['part_2'].fillna(0, inplace = True)
 test_df['part_3'].fillna(0, inplace = True)
 test_df['part_4'].fillna(0, inplace = True)
 test_df['part_5'].fillna(0, inplace = True)
 test_df['part_6'].fillna(0, inplace = True)
 test_df['part_7'].fillna(0, inplace = True)
 test_df['type_of_concept'].fillna(0, inplace = True)
 test_df['type_of_intention'].fillna(0, inplace = True)
 test_df['type_of_solving_question'].fillna(0, inplace = True)
 test_df['type_of_starter'].fillna(0, inplace = True)
 test_df['part_1_boolean'].fillna(0, inplace = True)
 test_df['part_2_boolean'].fillna(0, inplace = True)
 test_df['part_3_boolean'].fillna(0, inplace = True)
 test_df['part_4_boolean'].fillna(0, inplace = True)
 test_df['part_5_boolean'].fillna(0, inplace = True)
 test_df['part_6_boolean'].fillna(0, inplace = True)
 test_df['part_7_boolean'].fillna(0, inplace = True)
 test_df['type_of_concept_boolean'].fillna(0, inplace = True)
 test_df['type_of_intention_boolean'].fillna(0, inplace = True)
 test_df['type_of_solving_question_boolean'].fillna(0, inplace = True)
 test_df['type_of_starter_boolean'].fillna(0, inplace = True)
 test_df['answered_correctly_user'].fillna(0.65, inplace=True)
 test_df['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 test_df['quest_pct'].fillna(content_mean, inplace=True)
 test_df['part'] = test_df.part - 1
test_df['part'].fillna(4, inplace = True)
 test_df['avg_questions_seen'].fillna(1, inplace = True)
 test_df['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df["prior_question_had_explanation_enc"] = lb_make.fit_transform(test_df["prior_question_had_explanation"])
 test_df['answered_correctly'] = model.predict(test_df[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']])
 env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<set_options>
%%bash
cd LightGBM
rm -r build
mkdir build
cd build
cmake -DUSE_GPU=1 -DOpenCL_LIBRARY=/usr/local/cuda/lib64/libOpenCL.so -DOpenCL_INCLUDE_DIR=/usr/local/cuda/include/..
make -j$(nproc )
warnings.filterwarnings("ignore")
 _ = np.seterr(divide='ignore', invalid='ignore' )<define_variables>
!cd LightGBM/python-package/;python3 setup.py install --precompile
data_path = ".. /input/riiid-test-answer-prediction/train.csv"
questions_path = ".. /input/riiid-test-answer-prediction/questions.csv"<data_type_conversions>
!nvidia-smi
%%time
dtype = {
"row_id": "int64",
"timestamp": "int64",
"user_id": "int32",
"content_id": "int16",
"content_type_id": "boolean",
"task_container_id": "int16",
"user_answer": "int8",
"answered_correctly": "int8",
"prior_question_elapsed_time": "float32",
"prior_question_had_explanation": 'boolean'
}
train = pd.read_parquet(".. /input/riiid-parquet-files/train.parquet")
 train = train[dtype.keys() ]
train = train.astype(dtype)
 train = train[train['answered_correctly']!=-1]
train['prior_question_had_explanation'].fillna(False, inplace=True)
 train['prior_question_had_explanation'] = train['prior_question_had_explanation'].astype(bool)
 train = train[['user_id','content_id','answered_correctly',
'prior_question_elapsed_time', 'prior_question_had_explanation']]
train.info()<define_variables>
%matplotlib inline
pd.set_option('display.max_columns', 1000)
 warnings.filterwarnings('ignore')
 plt.style.use('ggplot')
 pd.set_option('display.max_rows', 1000)
 pd.set_option('display.max_columns', 1000)
TARGET = "answered_correctly"
TIME_MEAN = 21000.0
TIME_MIN = 0.0
TIME_MAX = 300000.0 
<load_from_csv>
%%time
df_train_raw = pd.read_csv('.. /input/train.csv')
 df_test_raw = pd.read_csv('.. /input/test.csv' )
dtype={'question_id':'int16','part':'int8','bundle_id':'int8', 'tags':'str'}
questions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',
usecols=dtype.keys() ,
dtype=dtype)
 questions['tags'].fillna('None', inplace=True)
 questions['num_tags'] = questions['tags'].apply(lambda x:len(x.split())if pd.notna(x)else 0)
 questions['tags'].fillna('None', inplace=True)
 le = LabelEncoder()
questions['tags_label'] = le.fit_transform(questions['tags'].values)
 questions = questions[['question_id','part','tags_label']]
questions.isnull().sum()<merge>
%%time
df_train = df_train_raw.copy()
df_test = df_test_raw.copy()
def preprocess(df):
df = df[df[TARGET] != -1].reset_index(drop=True)
 df = df.sort_values(['timestamp'], ascending=True ).reset_index(drop = True)
 df = df.merge(questions, left_on='content_id', right_on='question_id', how='left')
 df.drop(columns=['question_id'], inplace=True)
 df["prior_question_had_explanation"].fillna(False, inplace=True)
 df["prior_question_elapsed_time"] = df["prior_question_elapsed_time"].fillna(TIME_MEAN)
 df['lag'] = df.groupby('user_id')[TARGET].shift()
cum = df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 df['user_correctness'] = cum['cumsum'] / cum['cumcount']
df.drop(columns=['lag'], inplace=True)
 user_agg = df.groupby('user_id')[TARGET].agg(['sum', 'count'])
 content_agg = df.groupby('content_id')[TARGET].agg(['sum', 'count'])
 df['content_count'] = df['content_id'].map(content_agg['count'] ).astype('int32')
 df['content_mean'] = df['content_id'].map(content_agg['sum'] / content_agg['count'])
 return df<merge>
train_cols = [col for col in df_train.columns if col not in ['ID_code', 'target']]
y_train = df_train['target']
%%time
train['lag'] = train.groupby('user_id')[TARGET].shift()
cum = train.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 train['user_mean'] = cum['cumsum'] / cum['cumcount']
train.drop(columns=['lag'], inplace=True)
 user_agg = train.groupby('user_id')[TARGET].agg(['sum', 'count'])
 content_agg = train.groupby('content_id')[TARGET].agg(['sum', 'count'])
 train_df = train.groupby('user_id' ).tail(24 ).reset_index(drop=True)
 train_df = pd.merge(train_df, questions, left_on='content_id', right_on='question_id', how='left')
 train_df.drop(columns=['question_id'], inplace=True)
 train_df['content_count'] = train_df['content_id'].map(content_agg['count'] ).astype('int32')
 train_df['content_mean'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 valid_df = train_df.groupby('user_id' ).tail(6)
 train_df.drop(valid_df.index, inplace=True)
 del train<define_variables>
interactions= {'var_81':['var_53','var_139','var_12','var_76'],
'var_12':['var_139','var_26','var_22', 'var_53','var_110','var_13'],
'var_139':['var_146','var_26','var_53', 'var_6', 'var_118'],
'var_53':['var_110','var_6'],
'var_26':['var_110','var_109','var_12'],
'var_118':['var_156'],
'var_9':['var_89'],
'var_22':['var_28','var_99','var_26'],
'var_166':['var_110'],
'var_146':['var_40','var_0'],
'var_80':['var_12']}
FE = ['content_mean','content_count','user_mean',
'prior_question_elapsed_time','prior_question_had_explanation',
'part','tags_label']
CF = ['prior_question_had_explanation','part']<prepare_x_and_y>
%%time
for col in train_cols:
df_train[col+'_2'] = df_train[col] * df_train[col]
df_train[col+'_3'] = df_train[col] * df_train[col]* df_train[col]
df_test[col+'_2'] = df_test[col] * df_test[col]
df_test[col+'_3'] = df_test[col] * df_test[col]* df_test[col]
X_train, y_train = train_df[FE], train_df[TARGET]
X_test, y_test = valid_df[FE], valid_df[TARGET]<choose_model_class>
%%time
for df in [df_train, df_test]:
df['sum'] = df[train_cols].sum(axis=1)
 df['min'] = df[train_cols].min(axis=1)
 df['max'] = df[train_cols].max(axis=1)
 df['mean'] = df[train_cols].mean(axis=1)
 df['std'] = df[train_cols].std(axis=1)
 df['skew'] = df[train_cols].skew(axis=1)
 df['kurt'] = df[train_cols].kurtosis(axis=1)
 df['med'] = df[train_cols].median(axis=1 )
best_params = {
'iterations': 20000,
}
catb = CatBoostClassifier(**best_params,
loss_function='CrossEntropy',
eval_metric = 'AUC',
thread_count=2,
use_best_model=True,
task_type = "GPU",
verbose = False)
 roc_auc = list()
average_precision = list()
best_iteration = list()
X_train, y_train = train_df[FE], train_df[TARGET]
X_test, y_test = valid_df[FE], valid_df[TARGET]
train = Pool(data=X_train,
label=y_train,
feature_names=FE,
cat_features=[])
 test = Pool(data=X_test,
label=y_test,
feature_names=FE,
cat_features=[])
 catb.fit(train,
verbose_eval=50,
early_stopping_rounds=100,
eval_set=test,
use_best_model=True,
plot=False)
 best_iteration.append(catb.best_iteration_)
 preds = catb.predict_proba(X_test)
 roc_auc.append(roc_auc_score(y_true=y_test, y_score=preds[:,1]))
average_precision.append(average_precision_score(y_true=y_test, y_score=preds[:,1]))
print("Average cv roc auc score %0.3f ± %0.3f" %(np.mean(roc_auc), np.std(roc_auc)))
 print("Average cv roc average precision %0.3f ± %0.3f" %(np.mean(average_precision), np.std(average_precision)))
 catb.save_model('catb_model.cbm' )<data_type_conversions>
%%time
for key in interactions:
for value in interactions[key]:
df_train[key+'_'+value+'_mul'] = df_train[key]*df_train[value]
df_train[key+'_'+value+'_div'] = df_train[key]/df_train[value]
df_train[key+'_'+value+'_sum'] = df_train[key] + df_train[value]
df_train[key+'_'+value+'_sub'] = df_train[key] - df_train[value]
df_test[key+'_'+value+'_mul'] = df_test[key]*df_test[value]
df_test[key+'_'+value+'_div'] = df_test[key]/df_test[value]
df_test[key+'_'+value+'_sum'] = df_test[key] + df_test[value]
df_test[key+'_'+value+'_sub'] = df_test[key] - df_test[value]
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int))
content_sum_dict = content_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
content_count_dict = content_agg['count'].astype('int32' ).to_dict(defaultdict(int))<split>
df_train['num_zero_rows'] =(df_train_raw[train_cols] == 0 ).astype(int ).sum(axis=1)
 df_test['num_zero_rows'] =(df_test_raw[train_cols] == 0 ).astype(int ).sum(axis=1 )
env = riiideducation.make_env()
iter_test = env.iter_test()
prior_test_df = None<feature_engineering>
all_columns = [col for col in df_train.columns if col not in ['ID_code', 'target']]
%%time
for test_df, sample_prediction_df in iter_test:
if prior_test_df is not None:
prior_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[TARGET] != -1].reset_index(drop=True)
 user_ids = prior_test_df['user_id'].values
content_ids = prior_test_df['content_id'].values
targets = prior_test_df[TARGET].values
for user_id, content_id, target in zip(user_ids, content_ids, targets):
user_sum_dict[user_id] += target
user_count_dict[user_id] += 1
content_sum_dict[content_id] += target
content_count_dict[content_id] += 1
prior_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df = test_df.merge(questions, left_on='content_id', right_on='question_id', how='left')
 test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation']\ 
.fillna(False ).astype(bool)
 user_sum = np.zeros(len(test_df), dtype=np.int16)
 user_count = np.zeros(len(test_df), dtype=np.int16)
 content_sum = np.zeros(len(test_df), dtype=np.int32)
 content_count = np.zeros(len(test_df), dtype=np.int32)
 for i,(user_id, content_id)in enumerate(zip(test_df['user_id'].values, \
test_df['content_id'].values)) :
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
content_sum[i] = content_sum_dict[content_id]
content_count[i] = content_count_dict[content_id]
test_df['user_mean'] = user_sum / user_count
test_df['content_count'] = content_count
test_df['content_mean'] = content_sum / content_count
Xtest = test_df[FE].values
Xtest = Pool(data=Xtest,
feature_names=FE,
cat_features=[])
 test_df['answered_correctly'] = catb.predict_proba(Xtest)[:,1]
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])
<import_modules>
params = {
'num_leaves': 13,
'max_bin': 63,
'min_data_in_leaf': 80,
'learning_rate': 0.0081,
'min_sum_hessian_in_leaf': 10.0,
'bagging_fraction': 0.331,
'bagging_freq': 5,
'max_depth': -1,
'save_binary': True,
'feature_fraction': 0.041,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'auc',
'boost_from_average': False,
'device': 'gpu',
'gpu_platform_id':0,
'gpu_device_id': 0,
'seed':44000
}
num_round = 20000
import pandas as pd
import numpy as np
from sklearn.metrics import roc_auc_score<load_from_csv>
%%time
folds = StratifiedKFold(n_splits = 5, shuffle = True, random_state = 44000)
 oof = np.zeros(len(df_train))
predictions = np.zeros(len(df_test))
feature_import_df = pd.DataFrame()
for n_fold,(train_idx, val_idx)in enumerate(folds.split(df_train, y_train)) :
print("fold number =", n_fold+1)
 train_data = lgb.Dataset(df_train.iloc[train_idx][all_columns], label = y_train.iloc[train_idx])
 val_y = y_train.iloc[val_idx]
val_data = lgb.Dataset(df_train.iloc[val_idx][all_columns], label = val_y)
 watchlist = [train_data,val_data]
clf = lgb.train(params, train_data, num_boost_round = num_round,
valid_sets = watchlist, verbose_eval = 4000,
early_stopping_rounds=3000)
 oof[val_idx] = clf.predict(df_train.iloc[val_idx][all_columns], num_iteration=clf.best_iteration)
 fold_import_df = pd.DataFrame()
fold_import_df['Feature'] = all_columns
fold_import_df["importance"] = clf.feature_importance()
fold_import_df['fold'] = n_fold +1
feature_import_df = pd.concat([feature_import_df,fold_import_df], axis = 0)
 predictions += clf.predict(df_test[all_columns])/folds.n_splits
print("\tFold AUC Score: {}\tf1_score: {}
".format(roc_auc_score(val_y,oof[val_idx]),
f1_score(val_y,np.round(oof[val_idx]))))
gc.collect()
print("
CV AUC Score and std", roc_auc_score(y_train, oof),np.std(oof))
print("CV F1 Score", f1_score(y_train, np.round(oof)) )
train = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',
usecols=[1, 2, 3, 4, 5, 7, 8, 9],
dtype={'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id': 'int8',
'task_container_id': 'int16',
'answered_correctly':'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'boolean'}
)<load_from_csv>
sub = pd.DataFrame({'ID_code': df_test.ID_code.values,
'target': predictions})
 sub.to_csv('lgbm_0401_kernelgpu.csv', index = False )