Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<filter>
warnings.simplefilter(action='ignore', category=FutureWarning)
 train_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/train.csv')
 test_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/test.csv')
 synthetic_samples_indexes = np.load('.. /input/fakedata/synthetic_samples_indexes.npy')
 private_LB = np.load('.. /input/fakedata/private_LB.npy')
 public_LB = np.load('.. /input/fakedata/public_LB.npy')
 full = pd.concat([train_df, pd.concat([test_df.loc[private_LB], test_df.loc[public_LB]], sort = False)], sort = False)
 for feat in ['var_' + str(x)for x in range(200)]:
count_values = full.groupby(feat)[feat].count()
train_df['new_' + feat] = count_values.loc[train_df[feat]].values
test_df['new_' + feat] = count_values.loc[test_df[feat]].values
print('full_df has {} rows and {} columns'.format(full.shape[0], full.shape[1]))
positive_train = train_data[train_data['sentiment']=='positive']
neutral_train = train_data[train_data['sentiment']=='neutral']
negative_train = train_data[train_data['sentiment']=='negative']<define_variables>
seed = 2319
param = {
'num_leaves': 8,
'min_data_in_leaf': 17,
'learning_rate': 0.01,
'min_sum_hessian_in_leaf': 9.67,
'bagging_fraction': 0.8329,
'bagging_freq': 2,
'feature_fraction': 1,
'lambda_l1': 0.6426,
'lambda_l2': 0.3067,
'min_gain_to_split': 0.02832,
'max_depth': -1,
'seed': seed,
'feature_fraction_seed': seed,
'bagging_seed': seed,
'drop_seed': seed,
'data_random_seed': seed,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbosity': -1,
'metric': 'auc',
'is_unbalance': True,
'save_binary': True,
'boost_from_average': 'false',
'num_threads': 8
}
features = [c for c in train_df.columns if(c not in ['ID_code', 'target'])]
test_size = 0.3
X_train, X_test, y_train, y_test = train_test_split(train_df[features], train_df['target'], test_size = test_size, random_state=42)
 iterations = 110
y_hat = np.zeros([int(200000*test_size), 200])
 i = 0
for feature in ['var_' + str(x)for x in range(200)]:
feat_choices = [feature, 'new_' + feature]
lgb_train = lgb.Dataset(X_train[feat_choices], y_train)
 gbm = lgb.train(param, lgb_train, iterations, verbose_eval=-1)
 y_hat[:, i] = gbm.predict(X_test[feat_choices], num_iteration=gbm.best_iteration)
 i += 1
sub_preds =(y_hat ).sum(axis=1)
 score = roc_auc_score(y_test, sub_preds)
 print('Your CV score is', score )
max_len = 128
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file = '/kaggle/input/roberta-base/vocab.json',
merges_file = '/kaggle/input/roberta-base/merges.txt',
lowercase =True,
add_prefix_space=True 
)
 sentiment_id = {'positive':tokenizer.encode('positive' ).ids[0],
'negative':tokenizer.encode('negative' ).ids[0],
'neutral':tokenizer.encode('neutral' ).ids[0]}
train_data.reset_index(inplace=True)
 tot_tw = train_data.shape[0]
input_ids = np.ones(( tot_tw, max_len), dtype='int32')
 attention_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 token_type_ids = np.zeros(( tot_tw, max_len), dtype='int32')
 start_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 end_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 for i in range(tot_tw):
set1 = " "+" ".join(train_data.loc[i,'text'].split())
 set2 = " ".join(train_data.loc[i,'selected_text'].split())
 idx = set1.find(set2)
 set2_loc = np.zeros(( len(set1)))
 set2_loc[idx:idx+len(set2)]=1
if set1[idx-1]==" ":
set2_loc[idx-1]=1
enc_set1 = tokenizer.encode(set1)
 selected_text_token_idx=[]
for k,(a,b)in enumerate(enc_set1.offsets):
sm = np.sum(set2_loc[a:b])
 if sm > 0:
selected_text_token_idx.append(k)
 senti_token = sentiment_id[train_data.loc[i,'sentiment']]
input_ids[i,:len(enc_set1.ids)+5] = [0]+enc_set1.ids+[2,2]+[senti_token]+[2]
attention_mask[i,:len(enc_set1.ids)+5]=1
if len(selected_text_token_idx)> 0:
start_mask[i,selected_text_token_idx[0]+1]=1
end_mask[i, selected_text_token_idx[-1]+1]=1<compute_train_metric>
iterations = 126
param = {'bagging_fraction': 0.7693,
'bagging_freq': 2,
'lambda_l1': 0.7199,
'lambda_l2': 1.992,
'learning_rate': 0.009455,
'max_depth': 3,
'min_data_in_leaf': 22,
'min_gain_to_split': 0.06549,
'min_sum_hessian_in_leaf': 18.55,
'num_leaves': 20,
'feature_fraction': 1,
'save_binary': True,
'seed': 2319,
'feature_fraction_seed': 2319,
'bagging_seed': 2319,
'drop_seed': 2319,
'data_random_seed': 2319,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbosity': -1,
'metric': 'auc',
'is_unbalance': True,
'boost_from_average': 'false',
'num_threads': 6}
def custom_loss(y_true, y_pred):
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred, from_logits = False, label_smoothing = 0.20)
 loss = tf.reduce_mean(loss)
 return loss<load_pretrained>
iterations = 126
param = {'bagging_fraction': 0.7693,
'bagging_freq': 2,
'lambda_l1': 0.7199,
'lambda_l2': 1.992,
'learning_rate': 0.009455,
'max_depth': 3,
'min_data_in_leaf': 22,
'min_gain_to_split': 0.06549,
'min_sum_hessian_in_leaf': 18.55,
'num_leaves': 20,
'feature_fraction': 1,
'save_binary': True,
'seed': 2319,
'feature_fraction_seed': 2319,
'bagging_seed': 2319,
'drop_seed': 2319,
'data_random_seed': 2319,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbosity': -1,
'metric': 'auc',
'is_unbalance': True,
'boost_from_average': 'false',
'num_threads': 6}
features = [c for c in train_df.columns if(c not in ['ID_code', 'target'])]
test_size = 0.3
X_train, X_test, y_train, y_test = train_test_split(train_df[features], train_df['target'], test_size = test_size, random_state=42)
 iterations = 110
y_hat = np.zeros([int(200000*test_size), 200])
 i = 0
for feature in ['var_' + str(x)for x in range(200)]:
feat_choices = [feature, 'new_' + feature]
lgb_train = lgb.Dataset(X_train[feat_choices], y_train)
 gbm = lgb.train(param, lgb_train, iterations, verbose_eval=-1)
 y_hat[:, i] = gbm.predict(X_test[feat_choices], num_iteration=gbm.best_iteration)
 i += 1
def build_model() :
ids = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 att = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 config_path = RobertaConfig.from_pretrained('/kaggle/input/tf-roberta/config-roberta-base.json')
 roberta_model = TFRobertaModel.from_pretrained('/kaggle/input/tf-roberta/pretrained-roberta-base.h5', config=config_path)
 x = roberta_model(ids, attention_mask = att, token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.05 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.05 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss=custom_loss, optimizer=optimizer)
 return model<categorify>
weights = []
for col in range(200):
if roc_auc_score(y_test, y_hat[:,col])>= 0.5:
weights.append(roc_auc_score(y_test, y_hat[:,col]))
else:
weights.append(0)
 weights = np.array(weights)
 weights =(weights - weights.mean())/ weights.mean()
weights += 1
sub_preds_regular =(y_hat ).sum(axis=1)
 sub_preds_weighted =(y_hat*weights ).sum(axis=1)
 score_regular = roc_auc_score(y_test, sub_preds_regular)
 score_weighted = roc_auc_score(y_test, sub_preds_weighted)
 print('Your unweigthed score is:', score_regular)
 print('Your weigthed score is:', score_weighted )
tot_test_tw = test_data.shape[0]
input_ids_t = np.ones(( tot_test_tw,max_len), dtype='int32')
 attention_mask_t = np.zeros(( tot_test_tw,max_len), dtype='int32')
 token_type_ids_t = np.zeros(( tot_test_tw,max_len), dtype='int32')
 for i in range(tot_test_tw):
set1 = " "+" ".join(test_data.loc[i,'text'].split())
 enc_set1 = tokenizer.encode(set1)
 s_token = sentiment_id[test_data.loc[i,'sentiment']]
input_ids_t[i,:len(enc_set1.ids)+5]=[0]+enc_set1.ids+[2,2]+[s_token]+[2]
attention_mask_t[i,:len(enc_set1.ids)+5]=1<load_pretrained>
test_data_length = len(y_test)
 validation_length = int(test_data_length/4)
 weights = []
for col in range(200):
if roc_auc_score(y_test[:validation_length], y_hat[:validation_length,col])>= 0.5:
weights.append(roc_auc_score(y_test[:validation_length], y_hat[:validation_length,col]))
else:
weights.append(0)
 weights = np.array(weights)
 weights =(weights - weights.mean())/ weights.mean()
weights += 1
sub_preds_regular =(y_hat[validation_length:] ).sum(axis=1)
 sub_preds_weighted =(y_hat[validation_length:]*weights ).sum(axis=1)
 score_regular = roc_auc_score(y_test[validation_length:], sub_preds_regular)
 score_weighted = roc_auc_score(y_test[validation_length:], sub_preds_weighted)
 print('Your unweigthed score is:', score_regular)
 print('Your weigthed score is:', score_weighted)
 if score_weighted > score_regular:
print('Your weights ARE NOT overfitting')
 else:
print('Your weights ARE overfitting' )
pred_start= np.zeros(( input_ids_t.shape[0],max_len))
pred_end= np.zeros(( input_ids_t.shape[0],max_len))
for i in range(5):
print('--'*20)
 print('-- MODEL %i --'%(i+1))
print('--'*20)
 K.clear_session()
model = build_model()
model.load_weights('/kaggle/input/model4/v4-roberta-%i.h5'%(i))
pred = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=1)
 pred_start = pred_start +(pred[0]/5)
 pred_end = pred_end +(pred[1]/5 )<feature_engineering>
pd.options.mode.chained_assignment = None
min_n_unique_full = 2000
min_n_unique_train = int(min_n_unique_full*2/3)
 min_n_unique_test = int(min_n_unique_full*1/3)
 full = pd.concat([train_df, pd.concat([test_df.loc[private_LB], test_df.loc[public_LB]], sort = False)], sort = False)
 true_test_df = pd.concat([test_df.loc[private_LB], test_df.loc[public_LB]], sort = False)
 count = 1
for feat in ['var_' + str(x)for x in range(200)]:
if count%50 == 1:
print('Processing reached', feat)
 n_unique_list_full = list(set(full['new_' + feat]))
n_unique_list_full.sort()
for i in range(len(n_unique_list_full)) :
n_unique_full = n_unique_list_full[i]
len_n_unique_full = len(full[feat][full['new_' + feat] == n_unique_full])
 len_n_unique_train = len(train_df[feat][train_df['new_' + feat] == n_unique_full])
 len_n_unique_test = len(true_test_df[feat][true_test_df['new_' + feat] == n_unique_full])
 if len_n_unique_full < min_n_unique_full or len_n_unique_train < min_n_unique_train or len_n_unique_test < min_n_unique_test :
try:
full['new_' + feat][full['new_' + feat] == n_unique_full] = n_unique_list_full[i+1]
train_df['new_' + feat][train_df['new_' + feat] == n_unique_full] = n_unique_list_full[i+1]
true_test_df['new_' + feat][true_test_df['new_' + feat] == n_unique_full] = n_unique_list_full[i+1]
test_df['new_' + feat][test_df['new_' + feat] == n_unique_full] = n_unique_list_full[i+1]
except:
continue
for i in reversed(range(len(n_unique_list_full))):
n_unique_full = n_unique_list_full[i]
len_n_unique_full = len(full[feat][full['new_' + feat] == n_unique_full])
 len_n_unique_train = len(train_df[feat][train_df['new_' + feat] == n_unique_full])
 len_n_unique_test = len(true_test_df[feat][true_test_df['new_' + feat] == n_unique_full])
 if len_n_unique_full < min_n_unique_full or len_n_unique_train < min_n_unique_train or len_n_unique_test < min_n_unique_test :
try:
full['new_' + feat][full['new_' + feat] == n_unique_full] = n_unique_list_full[i-1]
train_df['new_' + feat][train_df['new_' + feat] == n_unique_full] = n_unique_list_full[i-1]
true_test_df['new_' + feat][true_test_df['new_' + feat] == n_unique_full] = n_unique_list_full[i-1]
test_df['new_' + feat][test_df['new_' + feat] == n_unique_full] = n_unique_list_full[i-1]
except:
continue
count+=1
print('Done!' )
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(pred_start[k,])
 b = np.argmax(pred_end[k,])
 if a>b:
st = test_data.loc[k,'text']
else:
text1 = " "+" ".join(test_data.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st)
 test_data['selected_text']=all
test_data.head(20 )<save_to_csv>
for n in [2, 53, 81, 111, 121, 126, 130, 146]:
print('Variable', 'var_' + str(n))
plt.figure(figsize=(15,8))
count = 1
for n_unique in list(set(train_df['new_var_' + str(n)])) [:6]:
var_tar_0 = train_df['var_' + str(n)][(train_df['new_var_' + str(n)] == n_unique)& 
(train_df['target'] == 0)]
var_tar_1 = train_df['var_' + str(n)][(train_df['new_var_' + str(n)] == n_unique)& 
(train_df['target'] == 1)]
var_tar_0
samples_0 = len(var_tar_0)
 samples_1 = len(var_tar_1)
 if samples_0 < 20 or samples_1 < 20:
continue
samples_percentage = np.round(( samples_0 + samples_1)*100/ 200000)
 plt.subplot(2, 3, count)
 sns.kdeplot(var_tar_0, shade=False, color="red", label = 'target = 0')
 sns.kdeplot(var_tar_1, shade=False, color="blue", label = 'target = 1')
 plt.title('Count = {} represents {}% of the data'.format(n_unique, samples_percentage))
plt.xlabel('Feature Values')
 plt.ylabel('Probability')
 count += 1
plt.tight_layout()
plt.show()
test_data[['textID','selected_text']].to_csv('submission.csv', index=False)
 print("Submission successful" )<load_from_csv>
for n in [117, 120]:
print('Variable', 'var_' + str(n))
plt.figure(figsize=(15,8))
count = 1
for n_unique in list(set(train_df['new_var_' + str(n)])) [:6]:
var_tar_0 = train_df['var_' + str(n)][(train_df['new_var_' + str(n)] == n_unique)& 
(train_df['target'] == 0)]
var_tar_1 = train_df['var_' + str(n)][(train_df['new_var_' + str(n)] == n_unique)& 
(train_df['target'] == 1)]
var_tar_0
samples_0 = len(var_tar_0)
 samples_1 = len(var_tar_1)
 if samples_0 < 20 or samples_1 < 20:
continue
samples_percentage = np.round(( samples_0 + samples_1)*100/ 200000)
 plt.subplot(2, 3, count)
 sns.kdeplot(var_tar_0, shade=False, color="red", label = 'target = 0')
 sns.kdeplot(var_tar_1, shade=False, color="blue", label = 'target = 1')
 plt.title('Count = {} represents {}% of the data'.format(n_unique, samples_percentage))
plt.xlabel('Feature Values')
 plt.ylabel('Probability')
 count += 1
plt.tight_layout()
plt.show()
train=pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 test=pd.read_csv("/kaggle/input/digit-recognizer/test.csv")
 train.head()<prepare_x_and_y>
n_unique = 1
n = 126
var_126_tar_0 = train_df['var_' + str(n)][(train_df['new_var_' + str(n)] == n_unique)& 
(train_df['target'] == 0)]
var_126_tar_1 = train_df['var_' + str(n)][(train_df['new_var_' + str(n)] == n_unique)& 
(train_df['target'] == 1)]
n = 81
var_81_tar_0 = train_df['var_' + str(n)][(train_df['new_var_' + str(n)] == n_unique)& 
(train_df['target'] == 0)]
var_81_tar_1 = train_df['var_' + str(n)][(train_df['new_var_' + str(n)] == n_unique)& 
(train_df['target'] == 1)]
fig = plt.subplots()
sns.kdeplot(var_126_tar_0, shade=False, color="r", label = 'var_126_tar_0')
 sns.kdeplot(var_126_tar_1, shade=False, color="blue", label = 'var_126_tar_1')
 sns.kdeplot(var_81_tar_0, shade=False, color="orange", label = 'var_81_tar_0')
 sns.kdeplot(var_81_tar_1, shade=False, color="black", label = 'var_81_tar_1')
 plt.title('PDFs of VAR 126 and 81 BEFORE transformation')
 plt.xlabel('Feature Values')
 plt.ylabel('Probability')
 plt.show()
n_unique = 1
n = 126
var_126_tar_0 = train_df['var_' + str(n)][(train_df['new_var_' + str(n)] == n_unique)& 
(train_df['target'] == 0)] * 2.486 -18.5
var_126_tar_1 = train_df['var_' + str(n)][(train_df['new_var_' + str(n)] == n_unique)& 
(train_df['target'] == 1)] * 2.486 - 18.5
n = 81
var_81_tar_0 = train_df['var_' + str(n)][(train_df['new_var_' + str(n)] == n_unique)& 
(train_df['target'] == 0)]
var_81_tar_1 = train_df['var_' + str(n)][(train_df['new_var_' + str(n)] == n_unique)& 
(train_df['target'] == 1)]
fig = plt.subplots()
sns.kdeplot(var_126_tar_0, shade=False, color="r", label = 'var_126_tar_0')
 sns.kdeplot(var_126_tar_1, shade=False, color="blue", label = 'var_126_tar_1')
 sns.kdeplot(var_81_tar_0, shade=False, color="orange", label = 'var_81_tar_0')
 sns.kdeplot(var_81_tar_1, shade=False, color="black", label = 'var_81_tar_1')
 plt.title('PDFs of VAR 126 and 81 AFTER transformation')
 plt.xlabel('Feature Values')
 plt.ylabel('Probability')
 plt.show()
<prepare_x_and_y><EOS> 
param = {'bagging_fraction': 0.5166,
'bagging_freq': 3,
'lambda_l1': 3.968,
'lambda_l2': 1.263,
'learning_rate': 0.00141,
'max_depth': 3,
'min_data_in_leaf': 17,
'min_gain_to_split': 0.2525,
'min_sum_hessian_in_leaf': 19.55,
'num_leaves': 20,
'feature_fraction': 1,
'save_binary': True,
'seed': 2319,
'feature_fraction_seed': 2319,
'bagging_seed': 2319,
'drop_seed': 2319,
'data_random_seed': 2319,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbosity': -1,
'metric': 'auc',
'is_unbalance': True,
'boost_from_average': 'false',
'num_threads': 6}
folds = StratifiedKFold(n_splits=4, shuffle=False, random_state=2319)
 target = train_df['target']
y_hat = np.zeros([200000, 200])
 test_hat = np.zeros([200000, 200])
 i = 0
for feature in ['var_' + str(x)for x in range(200)]:
print(feature)
 feat_choices = [feature, 'new_' + feature]
oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_df[feat_choices].values, target.values)) :
trn_data = lgb.Dataset(train_df.iloc[trn_idx][feat_choices], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(train_df.iloc[val_idx][feat_choices], label=target.iloc[val_idx])
 clf = lgb.train(param, trn_data, 126, valid_sets = [trn_data, val_data], verbose_eval=-1)
 oof[val_idx] = clf.predict(train_df.iloc[val_idx][feat_choices], num_iteration=clf.best_iteration)
 predictions += clf.predict(test_df[feat_choices], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)))
 y_hat[:, i] = oof
test_hat[:, i] = predictions
i += 1
weights = []
for col in range(200):
if roc_auc_score(target, y_hat[:,col])>= 0.5:
weights.append(roc_auc_score(target, y_hat[:,col]))
else:
weights.append(0)
 weights = np.array(weights)
 weights =(weights - weights.mean())/ weights.mean()
weights += 1
sub_preds =(y_hat*weights ).sum(axis=1)/200
print('Your CV score is', roc_auc_score(target, sub_preds))
sub_preds_test =(test_hat*weights ).sum(axis=1)/200
sub = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub["target"] = sub_preds_test
sub.to_csv('submission.csv', index=False )