Kaggle competition: Santander Customer Transaction Prediction
train_df['content_count'] = train_df['content_id'].map(content_agg['count'] ).astype('int32')
 train_df['content_id'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'] )<drop_column>
data = pd.read_csv('/kaggle/input/santander-customer-transaction-prediction/train.csv')
 data.info()
valid_df = train_df.groupby('user_id' ).tail(6)
 train_df.drop(valid_df.index, inplace=True )<init_hyperparams>
missing = False
for i in data.columns:
if len(pd.notnull(data[i])==True)!=len(data):
print(i, ": ",len(pd.notnull(data[i])))
 missing = True
if not missing:
print("No missing Data in train data." )
features = [
'content_id',
'prior_question_elapsed_time',
'prior_question_had_explanation',
'user_correctness',
'part',
'content_count'
]
params = {
'objective': 'binary',
'seed': 42,
'metric': 'auc',
'learning_rate': 0.05,
'max_bin': 800,
'num_leaves': 80
}<train_model>
test = pd.read_csv('.. /input/santander-customer-transaction-prediction/test.csv')
 test.head()
tr_data = lgb.Dataset(train_df[features], label=train_df[target])
 va_data = lgb.Dataset(valid_df[features], label=valid_df[target])
 model = lgb.train(
 params,
tr_data,
num_boost_round=10000,
valid_sets=[tr_data, va_data],
early_stopping_rounds=50,
verbose_eval=50 
)
 lgb.plot_importance(model, importance_type='gain')
 plt.show()<data_type_conversions>
missing = False
for i in test.columns:
if len(pd.notnull(test[i])==True)!=len(test):
print(i, ": ",len(pd.notnull(test[i])))
 missing = True
if not missing:
print("No missing Data in test." )
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int))
content_sum_dict = content_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
content_count_dict = content_agg['count'].astype('int32' ).to_dict(defaultdict(int))<prepare_output>
str(np.round(len(data[data['target']==1])/(len(data[data['target']==0])+ len(data[data['target']==0])) *100,2)) +' %, ratio of positive class'
%%time
for(test_df, sample_prediction_df)in iter_test:
if prior_test_df is not None:
prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)
 user_ids = prior_test_df['user_id'].values
content_ids = prior_test_df['content_id'].values
targets = prior_test_df[target].values
for user_id, content_id, answered_correctly in zip(user_ids, content_ids, targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
content_sum_dict[content_id] += answered_correctly
content_count_dict[content_id] += 1
prior_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')
 test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(False ).astype('bool')
 user_sum = np.zeros(len(test_df), dtype=np.int16)
 user_count = np.zeros(len(test_df), dtype=np.int16)
 content_sum = np.zeros(len(test_df), dtype=np.int32)
 content_count = np.zeros(len(test_df), dtype=np.int32)
 for i,(user_id, content_id)in enumerate(zip(test_df['user_id'].values, test_df['content_id'].values)) :
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
content_sum[i] = content_sum_dict[content_id]
content_count[i] = content_count_dict[content_id]
test_df['user_correctness'] = user_sum / user_count
test_df['content_count'] = content_count
test_df['content_id'] = content_sum / content_count
test_df[target] = model.predict(test_df[features])
 env.predict(test_df[['row_id', target]] )<import_modules>
stat_table["var"] = stat_table.apply(lambda x: np.abs(x["Class 0"] - x["Class 1"])>0.05 , 1)
 stat_table = stat_table[stat_table['var']==True]
import gc
import json
import pandas as pd
from pathlib import Path
import sqlite3
import riiideducation
import time
import xgboost as xgb<split>
data = all_data.iloc[:len(data),:].drop('ID_code', 1)
 test = all_data.iloc[len(data):,:].drop('target', 1 )
env = riiideducation.make_env()
iter_test = env.iter_test()<define_variables>
data['average_row'] = data[init_cols].apply(lambda x: np.round(np.mean(x),5), 1)
 test['average_row'] = test[init_cols].apply(lambda x: np.round(np.mean(x),5), 1 )
PATH = Path('.. /input/riiid-submission' )<choose_model_class>
data['std_row'] = data[init_cols].apply(lambda x: np.round(np.std(x),5), 1)
 test['std_row'] = test[init_cols].apply(lambda x: np.round(np.std(x),5), 1 )
model = xgb.Booster(model_file=PATH/'model.xgb')
 print('model loaded' )<define_variables>
all_data = pd.concat([data, test])
 for i in init_cols:
counts = all_data[i].value_counts()
all_data[i+"_count"] = all_data[i].map(counts )
dtypes = {
'answered_correctly': 'int8',
'answered_correctly_content_id_cumsum': 'int16',
'answered_correctly_content_id_cumsum_pct': 'int16',
'answered_correctly_cumsum': 'int16',
'answered_correctly_cumsum_pct': 'int8',
'answered_correctly_cumsum_upto': 'int8',
'answered_correctly_rollsum': 'int8',
'answered_correctly_rollsum_pct': 'int8',
'answered_incorrectly': 'int8',
'answered_incorrectly_content_id_cumsum': 'int16',
'answered_incorrectly_cumsum': 'int16',
'answered_incorrectly_rollsum': 'int8',
'bundle_id': 'uint16',
'content_id': 'int16',
'content_type_id': 'int8',
'correct_answer': 'uint8',
'lecture_id': 'uint16',
'lectures_cumcount': 'int16',
'part': 'uint8',
'part_correct_pct': 'uint8',
'prior_question_elapsed_time': 'float32',
'prior_question_elapsed_time_rollavg': 'float32',
'prior_question_had_explanation': 'bool',
'question_id': 'uint16',
'question_id_correct_pct': 'uint8',
'row_id': 'int64',
'tag': 'uint8',
'tag__0': 'uint8',
'tag__0_correct_pct': 'uint8',
'tags': 'str',
'task_container_id': 'int16',
'task_container_id_orig': 'int16',
'timestamp': 'int64',
'type_of': 'str',
'user_answer': 'int8',
'user_id': 'int32'
}
batch_cols_all = [
'user_id',
'content_id',
'row_id',
'task_container_id',
'timestamp',
'prior_question_elapsed_time',
'prior_question_had_explanation'
]
batch_cols_prior = [
'user_id',
'content_id',
'content_type_id'
]
with open(PATH/'columns.json')as cj:
test_cols = json.load(cj)
 batch_cols = ['user_id', 'content_id', 'row_id'] + [c for c in batch_cols_all if c in test_cols]
print('test_cols:')
 _ = list(map(print, test_cols))
dtypes_test = {k: v for k,v in dtypes.items() if k in test_cols}
dtypes_test = {**dtypes_test, **{'user_id': 'int32', 'content_id': 'int16'}}<load_pretrained>
powers = [2, 3]
for p in powers:
for i in init_cols:
all_data[i+"_"+str(p)] = np.power(all_data[i], p )
df_users_content = pd.read_pickle(PATH/'df_users_content.pkl')
 df_users_content.head()<data_type_conversions>
data = all_data.iloc[:len(data),:].drop('ID_code', 1)
 test = all_data.iloc[len(data):,:].drop('target', 1 )
df_users = df_users_content[['user_id', 'answered_correctly', 'answered_incorrectly']].groupby('user_id' ).sum().reset_index()
df_users = df_users.astype({'user_id': 'int32', 'answered_correctly': 'int16', 'answered_incorrectly': 'int16'})
 df_users.head()<load_pretrained>
del all_data
gc.collect()
df_questions = pd.read_pickle(PATH/'df_questions.pkl')
 df_questions.head()<create_dataframe>
X = subsampled_data[cols].values
y = subsampled_data['target'].values
scaler = MinMaxScaler()
scaler.fit(X)
 X = scaler.transform(X)
 X = TSNE(2, n_jobs=-1 ).fit_transform(X)
 df = pd.DataFrame()
df['target'] = y
df[['x', 'y']] = X
sns.scatterplot('x', 'y', 'target', data=df)
%%time
chunk_size = 20000
total = len(df_users_content)
 n_chunks =(total // chunk_size + 1)
 i = 0
while i < n_chunks:
df_users_content.iloc[i * chunk_size:(i + 1)* chunk_size].to_sql('users_content', conn, method='multi', if_exists='append', index=False)
 i += 1
conn.execute('CREATE UNIQUE INDEX users_content_index ON users_content(user_id, content_id)')
 del df_users_content
gc.collect()<load_from_csv>
submission = test.copy()
submission['target'] = 1
submission = submission [['ID_code', 'target']]
%%time
pd.read_sql('SELECT * from users_content LIMIT 5', conn )<drop_column>
from sklearn.preprocessing import StandardScaler
from collections import Counter
from sklearn.model_selection import StratifiedKFold
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import f1_score, roc_auc_score
from imblearn.over_sampling import RandomOverSampler
from sklearn.model_selection import GridSearchCV
%%time
chunk_size = 20000
total = len(df_users)
 n_chunks =(total // chunk_size + 1)
 i = 0
while i < n_chunks:
df_users.iloc[i * chunk_size:(i + 1)* chunk_size].to_sql('users', conn, method='multi', if_exists='append', index=False)
 i += 1
_ = conn.execute('CREATE UNIQUE INDEX users_index ON users(user_id)')
 del df_users
gc.collect()<load_from_csv>
X = data.drop('target', 1 ).values
y = data['target'].values
print('Original Dataset', Counter(y))
scaler = StandardScaler()
scaler.fit(X)
 X = scaler.transform(X)
 nm = RandomOverSampler()
X, y = nm.fit_resample(X, y)
 print('Resampled dataset shape %s' % Counter(y))
grid={"C":np.logspace(-3,3,4)}
logreg=LogisticRegression()
logreg_cv=GridSearchCV(logreg,grid,cv=10)
 logreg_cv.fit(X,y)
 print("tuned hpyerparameters :(best parameters)",logreg_cv.best_params_)
 print("accuracy :",logreg_cv.best_score_)
 scores_f1 = []
scores_roc = []
C = logreg_cv.best_params_['C']
kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=1)
 for train_ix, test_ix in kfold.split(X, y):
train_X, test_X = X[train_ix], X[test_ix]
train_y, test_y = y[train_ix], y[test_ix]
clf = LogisticRegression(C=C,)
 clf.fit(train_X, train_y)
 y_hat = clf.predict(train_X)
 scores_f1.append(f1_score(train_y, y_hat))
y_hat = clf.predict_proba(train_X)[:,1]
scores_roc.append(roc_auc_score(train_y, y_hat))
print('F1 Score: %f, +/-%f'%(np.mean(scores_f1), np.std(scores_f1)))
 print('ROC AUC Score: %f, +/-%f'%(np.mean(scores_roc), np.std(scores_roc)))
 clf = LogisticRegression(C=C)
 clf.fit(X, y)
 coefs = np.abs(clf.coef_[0])
 indices = np.argsort(coefs)[::-1]
plt.figure()
plt.title("Feature importances(Logistic Regression)")
 plt.bar(range(15), coefs[indices[:15]],
color="r", align="center")
 plt.xticks(range(15), data.columns[indices[:15]], rotation=45, ha='right')
 plt.subplots_adjust(bottom=0.3 )
%%time
pd.read_sql('SELECT * from users LIMIT 5', conn )<create_dataframe>
X = data.drop('target', 1 ).values
y = data['target'].values
print('Original Dataset', Counter(y))
nm = RandomOverSampler()
X, y = nm.fit_resample(X, y)
 scores_f1 = []
scores_roc = []
kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=1)
 for train_ix, test_ix in kfold.split(X, y):
train_X, test_X = X[train_ix], X[test_ix]
train_y, test_y = y[train_ix], y[test_ix]
clf = GaussianNB()
clf.fit(train_X, train_y)
 y_hat = clf.predict(train_X)
 scores_f1.append(f1_score(train_y, y_hat))
y_hat = clf.predict_proba(train_X)[:,1]
scores_roc.append(roc_auc_score(train_y, y_hat))
print('F1 Score: %f, +/-%f'%(np.mean(scores_f1), np.std(scores_f1)))
 print('ROC AUC Score: %f, +/-%f'%(np.mean(scores_roc), np.std(scores_roc)))
 clf = GaussianNB()
clf.fit(X, y)
 y_hat = clf.predict_proba(X)[:,1]
print('ROC AUC Score: %f'%(roc_auc_score(y, y_hat)))
%%time
q_cols = [
'question_id',
'part',
'tag__0',
'part_correct_pct',
'tag__0_correct_pct',
'question_id_correct_pct'
]
df_questions[q_cols].to_sql('questions', conn, method='multi', index=False)
 _ = conn.execute('CREATE UNIQUE INDEX question_id_index ON questions(question_id)')
 del df_questions
gc.collect()<load_from_csv>
gbdt_param = {
'objective': 'binary',
'boosting': 'gbdt',
'learning_rate': 0.01,
'num_leaves': 15,
'tree_learner': 'serial',
'num_threads': 8,
'seed': 42,
'max_depth': -1,
'min_data_in_leaf': 50,
'min_sum_hessian_in_leaf': 10,
'bagging_fraction': 0.6,
'bagging_freq': 5,
'feature_fraction': 0.05,
'lambda_l1': 1.,
'bagging_seed': 42,
'verbosity ': 1,
'boost_from_average': False,
'metric': 'auc',
}
X = data.drop('target', 1 ).values
y = data['target'].values
print('Original Dataset', Counter(y))
nm = RandomOverSampler()
X, y = nm.fit_resample(X, y)
 scores_f1 = []
scores_roc = []
kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=1)
 for train_ix, test_ix in kfold.split(X, y):
train_X, test_X = X[train_ix], X[test_ix]
train_y, test_y = y[train_ix], y[test_ix]
evals_result = {}
trn_data = lgb.Dataset(train_X, label=train_y)
 val_data = lgb.Dataset(test_X, label=test_y)
 lgb_clf = lgb.train(gbdt_param, trn_data, 10000,
valid_sets=[trn_data, val_data], early_stopping_rounds=5000,
verbose_eval=1000, evals_result=evals_result)
%%time
pd.read_sql('SELECT * from questions LIMIT 5', conn )<load_from_csv>
gbdt_param = {
'objective': 'binary',
'boosting': 'gbdt',
'learning_rate': 0.01,
'num_leaves': 15,
'tree_learner': 'serial',
'num_threads': 8,
'seed': 42,
'max_depth': -1,
'min_data_in_leaf': 50,
'min_sum_hessian_in_leaf': 10,
'bagging_fraction': 0.6,
'bagging_freq': 5,
'feature_fraction': 0.05,
'lambda_l1': 1.,
'bagging_seed': 42,
'verbosity ': 1,
'boost_from_average': False,
'metric': 'auc',
}
X = data.drop('target', 1 ).values
y = data['target'].values
nm = RandomOverSampler()
X, y = nm.fit_resample(X, y)
 trn_data = lgb.Dataset(X, label=y)
 lgb_clf = lgb.train(gbdt_param, trn_data, 100000)
<categorify><EOS> 
submission = test.copy()
submission['target'] = lgb_clf.predict(submission.drop('ID_code', 1 ).values)
 submission = submission [['ID_code', 'target']]
submission.to_csv('submission.csv', index=False )