Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<define_variables>
pd.options.display.max_rows = 200
pd.options.display.max_columns = 200
sns.set(style="darkgrid")
 warnings.filterwarnings('ignore')
 SEED = 42
max_len = 128
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file = '/kaggle/input/roberta-base/vocab.json',
merges_file = '/kaggle/input/roberta-base/merges.txt',
lowercase =True,
add_prefix_space=True 
)
 sentiment_id = {'positive':tokenizer.encode('positive' ).ids[0],
'negative':tokenizer.encode('negative' ).ids[0],
'neutral':tokenizer.encode('neutral' ).ids[0]}
train_data.reset_index(inplace=True)
 tot_tw = train_data.shape[0]
input_ids = np.ones(( tot_tw, max_len), dtype='int32')
 attention_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 token_type_ids = np.zeros(( tot_tw, max_len), dtype='int32')
 start_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 end_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 for i in range(tot_tw):
set1 = " "+" ".join(train_data.loc[i,'text'].split())
 set2 = " ".join(train_data.loc[i,'selected_text'].split())
 idx = set1.find(set2)
 set2_loc = np.zeros(( len(set1)))
 set2_loc[idx:idx+len(set2)]=1
if set1[idx-1]==" ":
set2_loc[idx-1]=1
enc_set1 = tokenizer.encode(set1)
 selected_text_token_idx=[]
for k,(a,b)in enumerate(enc_set1.offsets):
sm = np.sum(set2_loc[a:b])
 if sm > 0:
selected_text_token_idx.append(k)
 senti_token = sentiment_id[train_data.loc[i,'sentiment']]
input_ids[i,:len(enc_set1.ids)+5] = [0]+enc_set1.ids+[2,2]+[senti_token]+[2]
attention_mask[i,:len(enc_set1.ids)+5]=1
if len(selected_text_token_idx)> 0:
start_mask[i,selected_text_token_idx[0]+1]=1
end_mask[i, selected_text_token_idx[-1]+1]=1<compute_train_metric>
df_train = pd.read_csv('.. /input/train.csv')
 df_train.name = 'Training Set'
df_test = pd.read_csv('.. /input/test.csv')
 df_test.name = 'Test Set'
print('Number of Training Examples = {}'.format(df_train.shape[0]))
print('Number of Test Examples = {}'.format(df_test.shape[0]))
print('Training X Shape = {}'.format(df_train.shape))
print('Training y Shape = {}'.format(df_train['target'].shape[0]))
print('Test X Shape = {}'.format(df_test.shape))
print('Test y Shape = {}'.format(df_test.shape[0]))
print(df_train.columns)
 print(df_test.columns )
def custom_loss(y_true, y_pred):
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred, from_logits = False, label_smoothing = 0.20)
 loss = tf.reduce_mean(loss)
 return loss<load_pretrained>
df_train_unique = df_train.agg(['nunique'] ).transpose().sort_values(by='nunique')
 df_test_unique = df_test.agg(['nunique'] ).transpose().sort_values(by='nunique')
 df_uniques = df_train_unique.drop('target' ).reset_index().merge(df_test_unique.reset_index() , how='left', right_index=True, left_index=True)
 df_uniques.drop(columns=['index_y'], inplace=True)
 df_uniques.columns = ['Feature', 'Training Set Unique Count', 'Test Set Unique Count']
def build_model() :
ids = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 att = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 config_path = RobertaConfig.from_pretrained('/kaggle/input/tf-roberta/config-roberta-base.json')
 roberta_model = TFRobertaModel.from_pretrained('/kaggle/input/tf-roberta/pretrained-roberta-base.h5', config=config_path)
 x = roberta_model(ids, attention_mask = att, token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.05 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.05 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss=custom_loss, optimizer=optimizer)
 return model<categorify>
df_qdist = pd.DataFrame(np.zeros(( 200, 9)) , columns=['Quartile 1 Positives', 'Quartile 2 Positives', 'Quartile 3 Positives', 'Quartile 4 Positives',
'Quartile 1 Positive Percentage', 'Quartile 2 Positive Percentage', 'Quartile 3 Positive Percentage', 'Quartile 4 Positive Percentage',
'Quartile Order'])
 features = [col for col in df_train.columns.values.tolist() if col.startswith('var')]
quartiles = np.arange(0, 1, 0.25)
 df_qdist.index = features
for i, feature in enumerate(features):
for j, quartile in enumerate(quartiles):
target_counts = df_train[np.logical_and(df_train[feature] >= df_train[feature].quantile(q=quartile),
df_train[feature] < df_train[feature].quantile(q=quartile + 0.25)) ].target.value_counts()
ones_per = target_counts[1] /(target_counts[0] + target_counts[1])* 100
df_qdist.iloc[i, j] = target_counts[1]
df_qdist.iloc[i, j + 4] = ones_per
pers = df_qdist.columns.tolist() [4:-1]
for i, index in enumerate(df_qdist.index):
order = df_qdist[pers].iloc[[i]].sort_values(by=index, ascending=False, axis=1 ).columns
order_str = ''.join([col[9] for col in order])
 df_qdist.iloc[i, 8] = order_str
df_qdist = df_qdist.round(2)
 df_qdist.head(10 )
tot_test_tw = test_data.shape[0]
input_ids_t = np.ones(( tot_test_tw,max_len), dtype='int32')
 attention_mask_t = np.zeros(( tot_test_tw,max_len), dtype='int32')
 token_type_ids_t = np.zeros(( tot_test_tw,max_len), dtype='int32')
 for i in range(tot_test_tw):
set1 = " "+" ".join(test_data.loc[i,'text'].split())
 enc_set1 = tokenizer.encode(set1)
 s_token = sentiment_id[test_data.loc[i,'sentiment']]
input_ids_t[i,:len(enc_set1.ids)+5]=[0]+enc_set1.ids+[2,2]+[s_token]+[2]
attention_mask_t[i,:len(enc_set1.ids)+5]=1<load_pretrained>
df_qdist[np.logical_or(df_qdist['Quartile Order'].str.startswith('2'), df_qdist['Quartile Order'].str.startswith('3')) ]
pred_start= np.zeros(( input_ids_t.shape[0],max_len))
pred_end= np.zeros(( input_ids_t.shape[0],max_len))
for i in range(5):
print('--'*20)
 print('-- MODEL %i --'%(i+1))
print('--'*20)
 K.clear_session()
model = build_model()
model.load_weights('/kaggle/input/model4/v4-roberta-%i.h5'%(i))
pred = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=1)
 pred_start = pred_start +(pred[0]/5)
 pred_end = pred_end +(pred[1]/5 )<feature_engineering>
test = df_test.drop(['ID_code'], axis=1 ).values
unique_count = np.zeros_like(test)
 for feature in range(test.shape[1]):
_, index, count = np.unique(test[:, feature], return_counts=True, return_index=True)
 unique_count[index[count == 1], feature] += 1
real_samples = np.argwhere(np.sum(unique_count, axis=1)> 0)[:, 0]
synth_samples = np.argwhere(np.sum(unique_count, axis=1)== 0)[:, 0]
print('Number of real samples in test set is {}'.format(len(real_samples)))
 print('Number of synthetic samples in test set is {}'.format(len(synth_samples)) )
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(pred_start[k,])
 b = np.argmax(pred_end[k,])
 if a>b:
st = test_data.loc[k,'text']
else:
text1 = " "+" ".join(test_data.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st)
 test_data['selected_text']=all
test_data.head(20 )<save_to_csv>
features = [col for col in df_train.columns if col.startswith('var')]
df_all = pd.concat([df_train, df_test.ix[real_samples]])
 for feature in features:
temp = df_all[feature].value_counts(dropna=True)
 df_train[feature + 'vc'] = df_train[feature].map(temp ).map(lambda x: min(10, x)).astype(np.uint8)
 df_test[feature + 'vc'] = df_test[feature].map(temp ).map(lambda x: min(10, x)).astype(np.uint8)
 df_train[feature + 'sum'] =(( df_train[feature] - df_all[feature].mean())* df_train[feature + 'vc'].map(lambda x: int(x > 1)) ).astype(np.float32)
 df_test[feature + 'sum'] =(( df_test[feature] - df_all[feature].mean())* df_test[feature + 'vc'].map(lambda x: int(x > 1)) ).astype(np.float32)
 df_train[feature + 'sum2'] =(( df_train[feature])* df_train[feature + 'vc'].map(lambda x: int(x > 2)) ).astype(np.float32)
 df_test[feature + 'sum2'] =(( df_test[feature])* df_test[feature + 'vc'].map(lambda x: int(x > 2)) ).astype(np.float32)
 df_train[feature + 'sum3'] =(( df_train[feature])* df_train[feature + 'vc'].map(lambda x: int(x > 4)) ).astype(np.float32)
 df_test[feature + 'sum3'] =(( df_test[feature])* df_test[feature + 'vc'].map(lambda x: int(x > 4)) ).astype(np.float32)
 print('Training set shape after creating magic features: {}'.format(df_train.shape))
print('Test set shape after creating magic features: {}'.format(df_test.shape))
test_data[['textID','selected_text']].to_csv('submission.csv', index=False)
 print("Submission successful" )<load_from_csv>
def augment(x, y, t=2):
xs, xn = [], []
for i in range(t // 2):
mask = y == 0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 featnum = x1.shape[1] // 200 - 1
for c in range(200):
np.random.shuffle(ids)
 x1[:, [c] + [200 + featnum * c + idc for idc in range(featnum)]] = x1[ids][:, [c] + [200 + featnum * c + idc for idc in range(featnum)]]
xn.append(x1)
 for i in range(t):
mask = y > 0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 featnum = x1.shape[1] // 200 - 1
for c in range(200):
np.random.shuffle(ids)
 x1[:, [c] + [200 + featnum * c + idc for idc in range(1)]] = x1[ids][:, [c] + [200 + featnum * c + idc for idc in range(1)]]
xs.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x, xs, xn])
 y = np.concatenate([y, ys, yn])
 return x, y
train_data = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv')
 test_data = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv' )<count_missing_values>
print(train_data.notnull().sum())
 print(train_data.isnull().sum() )<correct_missing_values>
class KMeansFeaturizer:
def __init__(self, k, target_scale=5.0, random_state=None):
self.k = k
self.target_scale = target_scale
self.random_state = random_state
self.encoder = OneHotEncoder(categories='auto' ).fit(np.array(range(k)).reshape(-1, 1))
def fit(self, X, y=None):
if y is None:
kmeans = KMeans(n_clusters=self.k, n_init=20, random_state=self.random_state)
 kmeans.fit(X)
 self.kmeans = kmeans
self.cluster_centers_ = kmeans.cluster_centers_
else:
Xy = np.hstack(( X, y[:, np.newaxis] * self.target_scale))
kmeans_pretrain = KMeans(n_clusters=self.k, n_init=20, random_state=self.random_state)
 kmeans_pretrain.fit(Xy)
 kmeans = KMeans(n_clusters=self.k, init=kmeans_pretrain.cluster_centers_[:, :2], n_init=1, max_iter=1)
 kmeans.fit(X)
 self.kmeans = kmeans
self.cluster_centers_ = km_model.cluster_centers_
return self
def transform(self, X, y=None):
clusters = self.kmeans.predict(X)
 return self.encoder.transform(clusters.reshape(-1, 1))
def fit_transform(self, X, y=None):
self.fit(X, y)
 return self.transform(X, y)
train_data.dropna(axis = 0,inplace=True )<count_missing_values>
transform_feature(df=df_train, feature='var_108', transformation=np.round, decimals=2 )
print(test_data.notnull().sum())
 print(test_data.isnull().sum() )<feature_engineering>
gbdt_param = {
'objective': 'binary',
'boosting': 'gbdt',
'learning_rate': 0.01,
'num_leaves': 15,
'tree_learner': 'serial',
'num_threads': 8,
'seed': SEED,
'max_depth': -1,
'min_data_in_leaf': 50,
'min_sum_hessian_in_leaf': 10,
'bagging_fraction': 0.6,
'bagging_freq': 5,
'feature_fraction': 0.05,
'lambda_l1': 1.,
'bagging_seed': SEED,
'verbosity ': 1,
'boost_from_average': False,
'metric': 'auc',
}
def remove_punctuation(text):
no_punct = "".join([c for c in text if c not in string.punctuation])
 return no_punct
train_data['s_text_clean'] = train_data['selected_text'].apply(str ).apply(lambda x: remove_punctuation(x.lower()))
train_data.head(20 )<feature_engineering>
predictors = df_train.columns.tolist() [2:]
X_test = df_test[predictors]
n_splits = 5
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)
 oof = df_train[['ID_code', 'target']]
oof['predict'] = 0
predictions = df_test[['ID_code']]
val_aucs = []
feature_importance_df = pd.DataFrame()
tokenizer = RegexpTokenizer(r'\w+')
 train_data['s_text_tokens'] = train_data['s_text_clean'].apply(str ).apply(lambda x: tokenizer.tokenize(x))
train_data.head(20 )<feature_engineering>
for fold,(train_ind, val_ind)in enumerate(skf.split(df_train, df_train.target.values)) :
X_train, y_train = df_train.iloc[train_ind][predictors], df_train.iloc[train_ind]['target']
X_valid, y_valid = df_train.iloc[val_ind][predictors], df_train.iloc[val_ind]['target']
N = 1
p_valid, yp = 0, 0
for i in range(N):
print('
Fold {} - N {}'.format(fold + 1, i + 1))
X_t, y_t = augment(X_train.values, y_train.values)
 weights = np.array([0.8] * X_t.shape[0])
 weights[:X_train.shape[0]] = 1.0
print('Shape of X_train after augment: {}
Shape of y_train after augment: {}'.format(X_t.shape, y_t.shape))
X_t = pd.DataFrame(X_t)
 X_t = X_t.add_prefix('var_')
 trn_data = lgb.Dataset(X_t, label=y_t, weight=weights)
 val_data = lgb.Dataset(X_valid, label=y_valid)
 evals_result = {}
lgb_clf = lgb.train(gbdt_param, trn_data, 100000, valid_sets=[trn_data, val_data], early_stopping_rounds=5000, verbose_eval=1000, evals_result=evals_result)
 p_valid += lgb_clf.predict(X_valid)
 yp += lgb_clf.predict(X_test)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = predictors
fold_importance_df["importance"] = lgb_clf.feature_importance()
fold_importance_df["fold"] = fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 oof['predict'][val_ind] = p_valid / N
val_score = roc_auc_score(y_valid, p_valid)
 val_aucs.append(val_score)
 predictions['fold{}'.format(fold + 1)] = yp / N
def remove_stopwords(text):
words = [w for w in text if(w not in stopwords.words('english')or w not in 'im')]
return words
train_data['s_text_tokens_NOTstop'] = train_data['s_text_tokens'].apply(lambda x: remove_stopwords(x))
train_data.head(20 )<feature_engineering>
mean_auc = np.mean(val_aucs)
 std_auc = np.std(val_aucs)
 all_auc = roc_auc_score(oof['target'], oof['predict'])
 print('Mean AUC: {}, std: {}.
 All AUC: {}.'.format(mean_auc, std_auc, all_auc))
<string_transform><EOS> 
predictions['target'] = np.mean(predictions[[col for col in predictions.columns if col not in ['ID_code', 'target']]].values, axis=1)
 predictions.to_csv('predictions.csv', index=None)
 sub_df = pd.DataFrame({"ID_code":df_test["ID_code"].values})
 sub_df["target"] = predictions['target']
sub_df.to_csv("lgb_submission.csv", index=False)
 oof.to_csv('lgb_oof.csv', index=False )