Kaggle competition: Santander Customer Transaction Prediction
%%time
tuned_model = True
if tuned_model == False:
param_grid = {'eta': [0.05, 0.1, 0.2, 0.3]}
num_boost_round = 115
cvres = xbg_GridSearchCV(xgb, params, param_grid, dtrain, num_boost_round )<train_model>
train = pd.read_csv('.. /input/santander-customer-transaction-prediction/train.csv')
 test = pd.read_csv('.. /input/santander-customer-transaction-prediction/test.csv')
 train_knowledge = pd.read_csv('.. /input/santander-2019-distillation/lgbm_train.csv' )
%%time
tuned_model = False
if tuned_model == False:
xgbReg = XGBRegressor(**params, n_jobs=-1, n_estimators = 50)
 model_name = 'XGBoost_Regressor_with_100K'
features = [X_train_sample,X_val,test]
target = [y_train_sample, y_val]
pred = run_model(xgbReg, model_name, features, target )<choose_model_class>
y = train['target']
y_knowledge = train_knowledge['target']
id_code_train = train['ID_code']
id_code_test = test['ID_code']
features = [c for c in train.columns if c not in ['ID_code', 'target']]
%%time
tuned_model = False
if tuned_model == False:
xgbReg = XGBRegressor(**params, n_jobs=-1, n_estimators =50)
 model_name = 'XGBoost_Regressor_with_6M'
features = [X_train,X_val,test]
target = [y_train, y_val]
pred = run_model(xgbReg, model_name, features, target, usecv = False )<import_modules>
for feature in features:
train['sq_'+feature] =(train[feature])**2
train['c_'+feature] =(train[feature])**3
train['r2_'+feature] = np.round(train[feature], 2)
import math
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error<load_from_csv>
for feature in features:
test['sq_'+feature] =(test[feature])**2
test['c_'+feature] =(test[feature])**3
test['r2_'+feature] = np.round(test[feature], 2 )
df_train = pd.read_csv('.. /input/train.csv', nrows=2000000, parse_dates=['pickup_datetime'])
 df_test = pd.read_csv('.. /input/test.csv', parse_dates=['pickup_datetime'] )<concatenate>
class GaussRankScaler() :
def __init__(self):
self.epsilon = 1e-9
self.lower = -1 + self.epsilon
self.upper = 1 - self.epsilon
self.range = self.upper - self.lower
def fit_transform(self, X):
i = np.argsort(X, axis = 0)
 j = np.argsort(i, axis = 0)
 assert(j.min() == 0 ).all()
assert(j.max() == len(j)- 1 ).all()
j_range = len(j)- 1
self.divider = j_range / self.range
transformed = j / self.divider
transformed = transformed - self.upper
transformed = scipy.special.erfinv(transformed)
 return transformed
df_train['datatype'], df_test['datatype'] = 'training', 'testing'
df_test.insert(1, 'fare_amount', 0)
 df_data = pd.concat([df_train, df_test], ignore_index=True )<train_model>
SPLIT = len(train)
 train = train.append(test)
 del test; gc.collect()
scaler = GaussRankScaler()
sc = StandardScaler()
for feat in tqdm(features):
train[feat] = sc.fit_transform(train[feat].values.reshape(-1, 1))
train[feat+'_r'] = rankdata(train[feat] ).astype('float32')
 train[feat+'_n'] = norm.cdf(train[feat] ).astype('float32')
 feats = [c for c in train.columns if c not in(['ID_code', 'target'] + features)]
for feat in tqdm(feats):
train[feat] = sc.fit_transform(train[feat].values.reshape(-1, 1))
train = train.drop(['target', 'ID_code'], axis=1)
 test = train[SPLIT:].values
train = train[:SPLIT].values
print('Done!!')
 print(train.shape)
def distance(lat1: float, lon1: float, lat2: float, lon2: float)-> float:

 angle = 0.017453292519943295
x = 0.5 - np.cos(( lat2 - lat1)* angle)/ 2 + np.cos(lat1 * angle)* np.cos(lat2 * angle)*(1 - np.cos(( lon2 - lon1)* angle)) / 2
return 0.6213712 * 12742 * np.arcsin(np.sqrt(x))<feature_engineering>
x_train, x_valid, y_train, y_valid, y_knowledge_train, y_knowledge_valid = train_test_split(train, y, y_knowledge, stratify=y, test_size=0.2, random_state=8 )
df_data['fare_amount'] = np.log1p(df_data['fare_amount'] )<feature_engineering>
function = 'relu'
def create_model(input_shape, n_out):
input_tensor = Input(shape=input_shape)
 x = Dense(16, activation=function )(input_tensor)
 x = Flatten()(x)
 out_put = Dense(n_out, activation='sigmoid' )(x)
 model = Model(input_tensor, out_put)
 return model
df_data['year'] = df_data['pickup_datetime'].dt.year
df_data['quarter'] = df_data['pickup_datetime'].dt.quarter
df_data['month'] = df_data['pickup_datetime'].dt.month
df_data['weekofyear'] = df_data['pickup_datetime'].dt.weekofyear
df_data['weekday'] = df_data['pickup_datetime'].dt.weekday
df_data['dayofweek'] = df_data['pickup_datetime'].dt.dayofweek
df_data['hour'] = df_data['pickup_datetime'].dt.hour<feature_engineering>
def auc(y_true, y_pred):
return tf.py_func(roc_auc_score,(y_true, y_pred), tf.double )
df_data['distance_euclidean'] = distance(df_data['pickup_latitude'], df_data['pickup_longitude'], \
df_data['dropoff_latitude'], df_data['dropoff_longitude'])
 df_data['distance_latitude'] = df_data['dropoff_latitude'] - df_data['pickup_latitude']
df_data['distance_longitude'] = df_data['dropoff_longitude'] - df_data['pickup_longitude']<feature_engineering>
gamma = 2.0
alpha=.25
epsilon = K.epsilon()
def focal_loss(y_true, y_pred):
pt_1 = y_pred * y_true
pt_1 = K.clip(pt_1, epsilon, 1-epsilon)
 CE_1 = -K.log(pt_1)
 FL_1 = alpha* K.pow(1-pt_1, gamma)* CE_1
pt_0 =(1-y_pred)*(1-y_true)
 pt_0 = K.clip(pt_0, epsilon, 1-epsilon)
 CE_0 = -K.log(pt_0)
 FL_0 =(1-alpha)* K.pow(1-pt_0, gamma)* CE_0
loss = K.sum(FL_1, axis=1)+ K.sum(FL_0, axis=1)
 return loss
nyc =(40.7128, -74.0060)
 jfk =(40.6413, -73.7781)
 ewr =(40.6895, -74.1745)
 df_data['distance_pickup_to_nyc'] = distance(df_data['pickup_latitude'], df_data['pickup_longitude'], nyc[0], nyc[1])
 df_data['distance_pickup_to_jfk'] = distance(df_data['pickup_latitude'], df_data['pickup_longitude'], jfk[0], jfk[1])
 df_data['distance_pickup_to_ewr'] = distance(df_data['pickup_latitude'], df_data['pickup_longitude'], ewr[0], ewr[1])
 df_data['distance_dropoff_to_nyc'] = distance(df_data['dropoff_latitude'], df_data['dropoff_longitude'], nyc[0], nyc[1])
 df_data['distance_dropoff_to_jfk'] = distance(df_data['dropoff_latitude'], df_data['dropoff_longitude'], jfk[0], jfk[1])
 df_data['distance_dropoff_to_ewr'] = distance(df_data['dropoff_latitude'], df_data['dropoff_longitude'], ewr[0], ewr[1] )<feature_engineering>
def mixup_data(x, y, alpha=1.0):
if alpha > 0:
lam = np.random.beta(alpha, alpha)
 else:
lam = 1
sample_size = x.shape[0]
index_array = np.arange(sample_size)
 np.random.shuffle(index_array)
 mixed_x = lam * x +(1 - lam)* x[index_array]
mixed_y =(lam * y)+(( 1 - lam)* y[index_array])
 return mixed_x, mixed_y
def make_batches(size, batch_size):
nb_batch = int(np.ceil(size/float(batch_size)))
 return [(i*batch_size, min(size,(i+1)*batch_size)) for i in range(0, nb_batch)]
def batch_generator(X,y,batch_size=128,shuffle=True,mixup=False):
y = np.array(y)
 sample_size = X.shape[0]
index_array = np.arange(sample_size)
 while True:
if shuffle:
np.random.shuffle(index_array)
 batches = make_batches(sample_size, batch_size)
 for batch_index,(batch_start, batch_end)in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
X_batch = X[batch_ids]
y_batch = y[batch_ids]
if mixup:
X_batch,y_batch = mixup_data(X_batch,y_batch,alpha=1.0)
 yield X_batch,y_batch
df_data['fare_per_mile'] = df_data['fare_amount'] / df_data['distance_euclidean']
df_data['fare_per_mile'] = df_data['fare_per_mile'].apply(lambda x: 0 if x == float('inf')else x)
 df_data['fare_per_mile'] = df_data['fare_per_mile'].fillna(0 )<correct_missing_values>
model = create_model(( NUM_FEATURES,1), 1)
 model.compile(loss='binary_crossentropy', optimizer='adam')
 model.summary()
checkpoint = ModelCheckpoint('feed_forward_model.h5', monitor='val_loss', verbose=1,
save_best_only=True, mode='min', save_weights_only = True)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=4,
verbose=1, mode='min', epsilon=0.0001)
 early = EarlyStopping(monitor="val_loss",
mode="min",
patience=9)
 callbacks_list = [checkpoint, reduceLROnPlat, early]
tr_gen = batch_generator(x_train,y_train,batch_size=BATCH_SIZE,shuffle=True,mixup=True)
 history = model.fit_generator(
 tr_gen,
steps_per_epoch=np.ceil(float(len(x_train)) / float(BATCH_SIZE)) ,
epochs=10,
verbose=1,
callbacks=callbacks_list,
validation_data=(x_valid, y_valid))
df_data = df_data.dropna()<categorify>
model.load_weights('feed_forward_model.h5')
 prediction = model.predict(x_valid, batch_size=512, verbose=1)
 roc_auc_score(y_valid, prediction )
df_data = pd.get_dummies(df_data, columns=['datatype'], drop_first=True )<prepare_x_and_y>
y_train = np.vstack(( y_train, y_knowledge_train)).T
y_valid = np.vstack(( y_valid, y_knowledge_valid)).T
print(y_train.shape)
 y_train[0]
x = df_data[df_data['datatype_training'] == 1].drop(['key', 'pickup_datetime', 'fare_amount', 'fare_per_mile', 'datatype_training'], axis=1)
 y = df_data[df_data['datatype_training'] == 1]['fare_amount']<split>
def knowledge_distillation_loss_withBE(y_true, y_pred, beta=0.1):
y_true, y_pred_teacher = y_true[: , :1], y_true[: , 1:]
y_pred, y_pred_stu = y_pred[: , :1], y_pred[: , 1:]
loss = beta*binary_crossentropy(y_true,y_pred)+(1-beta)*binary_crossentropy(y_pred_teacher, y_pred_stu)
 return loss
x_train, x_validate, y_train, y_validate = train_test_split(x, y, test_size=0.25, random_state=58 )<compute_train_metric>
def auc_2(y_true, y_pred):
y_true = y_true[:, :1]
y_pred = y_pred[:, :1]
return tf.py_func(roc_auc_score,(y_true, y_pred), tf.double)
 def auc_3(y_true, y_pred):
y_true = y_true[:, :1]
y_pred = y_pred[:, 1:]
return tf.py_func(roc_auc_score,(y_true, y_pred), tf.double )
model_forestreg = RandomForestRegressor(n_estimators=10, max_depth=20, min_samples_split=1000, random_state=58)
 model_forestreg.fit(x_train, y_train)
 model_forestreg_ypredict = model_forestreg.predict(x_validate)
 model_forestreg_rmse = mean_squared_error(y_validate, model_forestreg_ypredict)** 0.5
model_forestreg_cvscores = np.sqrt(np.abs(cross_val_score(model_forestreg, x, y, cv=5, scoring='neg_mean_squared_error')))
 print('random forest regression
root mean squared error: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_forestreg_rmse, model_forestreg_cvscores.mean() , 2 * model_forestreg_cvscores.std()))<predict_on_test>
model = create_model(( NUM_FEATURES,1), 2)
 model.compile(loss=knowledge_distillation_loss_withBE, optimizer='adam', metrics=[auc_2])
 checkpoint = ModelCheckpoint('student_model_BE.h5', monitor='val_auc_2', verbose=1,
save_best_only=True, mode='max', save_weights_only = True)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_auc_2', factor=0.5, patience=4,
verbose=1, mode='max', epsilon=0.0001)
 early = EarlyStopping(monitor="val_auc_2",
mode="max",
patience=9)
 callbacks_list = [checkpoint, reduceLROnPlat, early]
history = model.fit(x_train,y_train,
epochs=10,
batch_size = BATCH_SIZE,
validation_data=(x_valid, y_valid))
final_model = model_forestreg
x_test = df_data[df_data['datatype_training'] == 0].drop(['key', 'pickup_datetime', 'fare_amount', 'fare_per_mile', 'datatype_training'], axis=1)
 y_test = pd.DataFrame(np.expm1(final_model.predict(x_test)) , columns=['fare_amount'], index=df_data.loc[df_data['datatype_training'] == 0, 'key'] )<save_to_csv>
def knowledge_distillation_loss_withFL(y_true, y_pred, beta=0.1):
y_true, y_pred_teacher = y_true[: , :1], y_true[: , 1:]
y_pred, y_pred_stu = y_pred[: , :1], y_pred[: , 1:]
loss = beta*focal_loss(y_true,y_pred)+(1-beta)*binary_crossentropy(y_pred_teacher, y_pred_stu)
 return loss
out = pd.DataFrame({'key': y_test.index, 'fare_amount': y_test['fare_amount']})
 out.to_csv('submission.csv', index=False )<feature_engineering>
model = create_model(( NUM_FEATURES,1), 2)
 model.compile(loss=knowledge_distillation_loss_withFL, optimizer='adam', metrics=[auc_2, auc_3])
 checkpoint = ModelCheckpoint('student_model_FL.h5', monitor='val_auc_2', verbose=1,
save_best_only=True, mode='max', save_weights_only = True)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_auc_2', factor=0.5, patience=4,
verbose=1, mode='max', epsilon=0.0001)
 early = EarlyStopping(monitor="val_auc_2",
mode="max",
patience=9)
 callbacks_list = [checkpoint, reduceLROnPlat, early]
history = model.fit(x_train,y_train,
epochs=10,
batch_size = BATCH_SIZE,
callbacks=callbacks_list,
validation_data=(x_valid, y_valid))
print(tf.__version__)
 assert tf.__version__ >= "1.8" or tf.__version__ >= "1.10"
tf.logging.set_verbosity(tf.logging.INFO)
 CSV_COLUMNS = ['fare_amount', 'pickup_datetime','pickup_longitude','pickup_latitude',
'dropoff_longitude','dropoff_latitude', 'passenger_count', 'key']
LABEL_COLUMN = 'fare_amount'
TRAIN_LINES = 55423856
DEBUG=False 
<feature_engineering>
def sigmoid(x, derivative=False):
return x*(1-x)if derivative else 1/(1+np.exp(-x))
TEMPERATURE = 2
y_knowledge_logit = logit(y_knowledge)
 y_temperature = sigmoid(y_knowledge_logit/TEMPERATURE)
 x_train, x_valid, y_train, y_valid, y_knowledge_train, y_knowledge_valid = train_test_split(train, y, y_temperature,
stratify=y, test_size=0.2, random_state=8)
@contextmanager
def timer(name):
t0 = time.time()
yield
print(f'[{name}] done in {time.time() - t0:.0f} s' )<feature_engineering>
y_train = np.vstack(( y_train, y_knowledge_train)).T
y_valid = np.vstack(( y_valid, y_knowledge_valid)).T
print(y_train.shape)
 y_train[0]
PATH = '.. /input'
train_df = pd.read_csv(f'{PATH}/train.csv', nrows=100000)
 train_df['distance'] = np.sqrt(np.abs(train_df['pickup_longitude']-train_df['dropoff_longitude'])**2 +
np.abs(train_df['pickup_latitude']-train_df['dropoff_latitude'])**2)
 train_df.head()
train_df.describe()<define_variables>
model = create_model(( NUM_FEATURES,1), 2)
 model.compile(loss=knowledge_distillation_loss_withFL, optimizer='adam', metrics=[auc_2,auc_3])
 checkpoint = ModelCheckpoint('student_model_FL.h5', monitor='val_auc_2', verbose=1,
save_best_only=True, mode='max', save_weights_only = True)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_auc_2', factor=0.5, patience=4,
verbose=1, mode='max', epsilon=0.0001)
 early = EarlyStopping(monitor="val_auc_2",
mode="max",
patience=9)
 callbacks_list = [checkpoint, reduceLROnPlat, early]
history = model.fit(x_train,y_train,
epochs=10,
batch_size = 1024,
callbacks=callbacks_list,
validation_data=(x_valid, y_valid))
BATCH_SIZE=1
dataset = tf.contrib.data.make_csv_dataset(
 file_pattern=f'{PATH}/train.csv',
batch_size=BATCH_SIZE,
column_names=None,
column_defaults=None,
label_name='fare_amount',
select_columns=[1, 2, 3, 4, 5, 6, 7],
field_delim=',',
use_quote_delim=True,
na_value='',
header=True,
num_epochs=None,
shuffle=True,
shuffle_buffer_size=10000,
shuffle_seed=None,
prefetch_buffer_size=1,
num_parallel_reads=1,
sloppy=False,
num_rows_for_inference=100 
)
 if DEBUG:
next_element = dataset.make_one_shot_iterator().get_next()
with tf.Session() as sess:
features, label = sess.run(next_element)
 print("Features:
", features, "
Label:
", label)
<data_type_conversions>
num_fold = 5
folds = list(StratifiedKFold(n_splits=num_fold, shuffle=True, random_state=7 ).split(train, y))
y_test_pred_log = np.zeros(len(train))
y_train_pred_log = np.zeros(len(train))
print(y_test_pred_log.shape)
 print(y_train_pred_log.shape)
 score = []
for j,(train_idx, valid_idx)in enumerate(folds):
print('
===================FOLD=',j)
 x_train, x_valid = train[train_idx], train[valid_idx]
y_train, y_valid = y[train_idx], y[valid_idx]
y_knowledge_train, y_knowledge_valid = y_temperature[train_idx], y_temperature[valid_idx]
y_train = np.vstack(( y_train, y_knowledge_train)).T
y_valid = np.vstack(( y_valid, y_knowledge_valid)).T
model = create_model(( NUM_FEATURES,1), 2)
 model.compile(loss=knowledge_distillation_loss_withFL, optimizer='adam', metrics=[auc_2, auc_3])
 checkpoint = ModelCheckpoint('student_model_FL.h5', monitor='val_auc_2', verbose=1,
save_best_only=True, mode='max', save_weights_only = True)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_auc_2', factor=0.5, patience=4,
verbose=1, mode='max', epsilon=0.0001)
 early = EarlyStopping(monitor="val_auc_2",
mode="max",
patience=9)
 callbacks_list = [checkpoint, reduceLROnPlat, early]
history = model.fit(x_train,y_train,
epochs=100,
batch_size = BATCH_SIZE,
callbacks=callbacks_list,
validation_data=(x_valid, y_valid))
model.load_weights('student_model_FL.h5')
 prediction = model.predict(x_valid,
batch_size=512,
verbose=1)
 score.append(roc_auc_score(y_valid[:,0], prediction[:,1]))
prediction = model.predict(test,
batch_size=512,
verbose=1)
 y_test_pred_log += np.squeeze(prediction[:, 1])
 prediction = model.predict(x_valid,
batch_size=512,
verbose=1)
 y_train_pred_log[valid_idx] += np.squeeze(prediction[:, 1])
 del x_train, x_valid, y_train, y_valid, y_knowledge_train, y_knowledge_valid
gc.collect()
def pd_weekDay(year, month, day):
df = pd.DataFrame({'year': year,
'month': month,
'day': day})
 date_df = pd.to_datetime(df)
 return date_df.dt.weekday.astype(np.int32)
 def pd_dayofYear(year, month, day):
df = pd.DataFrame({'year': year,
'month': month,
'day': day})
 date_df = pd.to_datetime(df)
 return date_df.dt.dayofyear.astype(np.int32)
 if DEBUG:
years=np.array([2018, 2018, 2018])
 months=np.array([8, 11, 1])
 days=np.array([20, 6, 8])
 print(pd_dayofYear(years, months, days)) 
<categorify>
print("OOF score: ", roc_auc_score(y, y_train_pred_log/num_fold))
print("average {} folds score: ".format(num_fold), np.sum(score)/num_fold)
def tf_isAirport(latitude,longitude,airport_name='JFK'):
jfkcoord = tf.constant([-73.8352, -73.7401, 40.6195, 40.6659])
 ewrcoord = tf.constant([-74.1925, -74.1531, 40.6700, 40.7081])
 lgucoord = tf.constant([-73.8895, -73.8550, 40.7664, 40.7931])
 if airport_name=='JFK':
coord = jfkcoord
elif airport_name=='EWR':
coord = ewrcoord
elif airport_name=='LGU':
coord = lgucoord
else:
raise ValueError(f'Unknown NYC Airport {airport_name}')
 is_airport = \
tf.logical_and(
 tf.logical_and(
 tf.greater(longitude, coord[0]), tf.less(longitude, coord[1])
),
tf.logical_and(
 tf.greater(latitude, coord[2]), tf.less(latitude, coord[3])
)
)
 return is_airport 
<data_type_conversions>
submit = pd.read_csv('.. /input/santander-customer-transaction-prediction/sample_submission.csv')
 submit['ID_code'] = id_code_test
submit['target'] = y_test_pred_log/num_fold
submit.to_csv('submission.csv', index=False)
 submit.head()