Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<categorify>
import logging, os, pandas as pd, numpy as np
import matplotlib.pyplot as plt, seaborn as sns
import lightgbm as lgb
from sklearn.model_selection import StratifiedKFold, train_test_split
from sklearn.metrics import mean_squared_error
from sklearn.metrics import roc_auc_score, roc_curve
if validaten_flg:
target_df = pd.read_pickle(valid_pickle)
 if debug:
target_df = target_df[:10000]
iter_test = Iter_Valid(target_df,max_user=1000)
 predicted = []
def set_predict(df):
predicted.append(df)
 answered_correctly_sum_u_dict = defaultdict(int)
 count_u_dict = defaultdict(int)
 train = pd.read_pickle(train_pickle)[['user_id','answered_correctly','content_type_id']]
if debug:
train = train[:1000000]
train = train[train.content_type_id == False].reset_index(drop=True)
 update_user_feats(train, answered_correctly_sum_u_dict, count_u_dict)
 del train
else:
env = riiideducation.make_env()
iter_test = env.iter_test()
set_predict = env.predict<merge>
%%time
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
previous_test_df = None
for(test_df, sample_prediction_df)in iter_test:
if previous_test_df is not None:
previous_test_df[TARGET] = eval(test_df["prior_group_answers_correct"].iloc[0])
 update_user_feats(previous_test_df, answered_correctly_sum_u_dict, count_u_dict)
 previous_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df = add_user_feats_without_update(test_df, answered_correctly_sum_u_dict, count_u_dict)
 test_df = pd.merge(test_df, content_df, on='content_id', how="left")
 test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')
 test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False ).astype('int8')
 test_df['prior_question_elapsed_time_mean'] = test_df.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 test_df[TARGET] = model.predict(test_df[FEATS])
 set_predict(test_df[['row_id', TARGET]] )<compute_test_metric>
sns.countplot(train['target'])
 print(train.target.value_counts(normalize=True))
if validaten_flg:
y_true = target_df[target_df.content_type_id == 0].answered_correctly
y_pred = pd.concat(predicted ).answered_correctly
print(roc_auc_score(y_true, y_pred))<install_modules>
def dupl_max_df(df):
dupl_max = []
for feature in features:
values = df[feature].value_counts()
dupl_max.append([feature, values.max() , values.idxmax() ])
 dupl_max = pd.DataFrame(dupl_max, columns=['feature', '
return dupl_max
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1<set_options>
dupl_max_df(train )
_ = np.seterr(divide='ignore', invalid='ignore' )<define_variables>
dupl_max_df(test )
data_types_dict = {
'user_id': 'int32',
'content_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'<load_from_csv>
%%time
for df in [train, test]:
df['sum'] = df[features].sum(axis=1)
 df['min'] = df[features].min(axis=1)
 df['max'] = df[features].max(axis=1)
 df['mean'] = df[features].mean(axis=1)
 df['std'] = df[features].std(axis=1)
 df['skew'] = df[features].skew(axis=1)
 df['kurt'] = df[features].kurtosis(axis=1)
 df['med'] = df[features].median(axis=1 )
train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()
train_df = train_df[train_df[target] != -1].reset_index(drop=True)
 train_df['prior_question_had_explanation'].fillna(False, inplace=True)
 train_df = train_df.astype(data_types_dict )<groupby>
features = [c for c in train.columns.values if c not in ['ID_code', 'target']]
target = train['target']
params = {
'bagging_freq': 2,
'bagging_fraction': 0.8,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.1403,
'learning_rate': 0.07,
'max_depth': 7,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 19,
'num_leaves': 14,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': -1,
'lambda_l1': 1.7916,
'lambda_l2': 4.7454,
'min_gain_to_split': 0.0319
}
train_df['lag'] = train_df.groupby('user_id')[target].shift()
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df.drop(columns=['lag'], inplace=True )<groupby>
%%time
val_size = 0.3
X_train, X_val, y_train, y_val = train_test_split(train[features], train['target'], test_size = val_size, random_state=42)
 iterations = 100
val_pred = np.zeros([int(len(train)*val_size), len(features)])
 test_pred = np.zeros([200000, len(features)])
 i = 0
for feature in features:
print(feature)
 feat = [feature]
train_data = lgb.Dataset(X_train[feat], y_train)
 gbm = lgb.train(params, train_data, iterations, verbose_eval=-1)
 val_pred[:, i] = gbm.predict(X_val[feat], num_iteration=gbm.best_iteration)
 test_pred[:, i] = gbm.predict(test[feat], num_iteration=gbm.best_iteration)
 i += 1
score = roc_auc_score(y_val,(val_pred ).sum(axis=1))
print('CV score: ', score )
user_agg = train_df.groupby('user_id')[target].agg(['sum', 'count'])
 content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count'] )<load_from_csv>
%%time
folds = StratifiedKFold(n_splits=10, shuffle=False, random_state=42)
 oof = np.zeros(len(train))
pred = np.zeros(len(test))
feature_important_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train, train['target'])) :
print('Fold: {}'.format(fold_))
trn_data = lgb.Dataset(train.iloc[trn_idx][features], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(train.iloc[val_idx][features], label=target.iloc[val_idx])
 num_round = 100000
clf = lgb.train(params, trn_data, num_round, valid_sets=[trn_data, val_data], verbose_eval=1000, early_stopping_rounds=3000)
 oof[val_idx] = clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_important_df = pd.DataFrame()
fold_important_df['feature'] = features
fold_important_df['importance'] = clf.feature_importance()
fold_important_df['fold'] = fold_ + 1
feature_important_df = pd.concat([feature_important_df, fold_important_df], axis=0)
 pred = clf.predict(test[features], num_iteration=clf.best_iteration)/folds.n_splits
print('CV score: {:.5f}'.format(roc_auc_score(target, oof)) )
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 3],
dtype={'question_id': 'int16', 'part': 'int8'} 
)
 train_df = pd.merge(train_df, questions_df, left_on='content_id', right_on='question_id', how='left')
 train_df.drop(columns=['question_id'], inplace=True )<categorify>
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = pred
submission.to_csv('submission.csv', index=False )