Kaggle competition: Santander Customer Transaction Prediction
def feat_eng_func(features, label=None):
print("Feature Engineered Label:", label)
 features['pickup_year'] = tf.string_to_number(tf.substr(features['pickup_datetime'], 0, 4), tf.int32)
 features['pickup_month'] = tf.string_to_number(tf.substr(features['pickup_datetime'], 5, 2), tf.int32)
 features['pickup_day'] = tf.string_to_number(tf.substr(features['pickup_datetime'], 8, 2), tf.int32)
 features['pickup_hour'] = tf.string_to_number(tf.substr(features['pickup_datetime'], 11, 2), tf.int32)
 features['pickup_weekday'] = tf.py_func(pd_weekDay,
[features['pickup_year'], features['pickup_month'], features['pickup_day']],
tf.int32,
stateful=False,
name='Weekday' 
)
 features['night'] = tf.cast(
 tf.logical_or(tf.greater(features['pickup_hour'], 19), tf.less(features['pickup_hour'], 7)) ,
tf.float32)
 minlat = tf.constant(38.0)
 maxlat = tf.constant(42.0)
 minlon = tf.constant(-76.0)
 maxlon = tf.constant(-72.0)
 features['pickup_longitude'] = tf.clip_by_value(features['pickup_longitude'], minlon, maxlon)
 features['pickup_latitude'] = tf.clip_by_value(features['pickup_latitude'], minlat, maxlat)
 features['dropoff_longitude'] = tf.clip_by_value(features['dropoff_longitude'], minlon, maxlon)
 features['dropoff_latitude'] = tf.clip_by_value(features['dropoff_latitude'], minlat, maxlat)
 minpass = tf.constant(1.0)
 maxpass = tf.constant(6.0)
 features['passenger_count'] = tf.clip_by_value(tf.cast(features['passenger_count'], tf.float32), minpass, maxpass)
 if label != None:
minfare = tf.constant(3.0)
 maxfare = tf.constant(300.0)
 label = tf.clip_by_value(label, minfare, maxfare)
 features['longitude_dist'] = tf.abs(features['pickup_longitude'] - features['dropoff_longitude'])
 features['latitude_dist'] = tf.abs(features['pickup_latitude'] - features['dropoff_latitude'])
 features['distance'] = tf.sqrt(features['longitude_dist']**2 + features['latitude_dist']**2)
 features['pick_angle'] = angle_from_nyc(features['pickup_longitude'],features['pickup_latitude'])
 features['drop_angle'] = angle_from_nyc(features['dropoff_longitude'],features['dropoff_latitude'])
 features['angle'] = features['pick_angle'] - features['drop_angle']
if label == None:
return features
return(features, label )<define_variables>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 features = [c for c in train.columns if c not in ['ID_code', 'target']]
target = train['target']
print("Data is ready!" )
def read_dataset(filename, mode, batch_size = 512):
def _input_fn() :
if mode == tf.estimator.ModeKeys.TRAIN:
num_epochs = None
shuffle = False
else:
num_epochs = 1
shuffle = False
if mode == tf.estimator.ModeKeys.PREDICT:
label_name=None
select_columns=[1, 2, 3, 4, 5, 6]
else:
label_name ='fare_amount'
select_columns = [1, 2, 3, 4, 5, 6, 7]
file_list = tf.gfile.Glob(filename)
 dataset = tf.contrib.data.make_csv_dataset(
 file_pattern=file_list,
batch_size=batch_size,
column_names=None,
column_defaults=None,
label_name=label_name,
select_columns=select_columns,
field_delim=',',
use_quote_delim=True,
na_value='',
header=True,
num_epochs=num_epochs,
shuffle=shuffle,
shuffle_buffer_size=128*batch_size,
shuffle_seed=None,
prefetch_buffer_size=1,
num_parallel_reads=1,
sloppy=False,
num_rows_for_inference=100 
)
 skip_train_lines = TRAIN_LINES // batch_size // 100 * 10
if mode == tf.estimator.ModeKeys.TRAIN:
dataset = dataset.skip(skip_train_lines)
 elif mode == tf.estimator.ModeKeys.EVAL:
dataset = dataset.take(skip_train_lines)
 dataset = dataset.map(feat_eng_func)
 return dataset.make_one_shot_iterator().get_next()
return _input_fn 
<load_from_csv>
print("Missing data at training")
 train.isnull().values.any()
if DEBUG:
train_input_fn = read_dataset(f'{PATH}/train.csv', tf.estimator.ModeKeys.EVAL, batch_size = 8)
 with timer('Debugging'):
with tf.Session() as sess:
features, label = sess.run(train_input_fn())
 print("Features:
", features, "
Label:
", label )<define_variables>
print("Missing data at test")
 test.isnull().values.any()
NUMERIC_COLUMNS = ['passenger_count', 'pickup_dense_year',
'longitude_dist', 'latitude_dist',
'distance', 'pick_dist_center', 'drop_dist_center', 'angle',
'pick_angle','drop_angle'
]
BOOL_COLUMNS = ['is_JFK_pickup', 'is_JFK_dropoff', 'is_EWR_pickup', 'is_EWR_dropoff',
'is_LGU_pickup', 'is_LGU_dropoff', 'is_NYC_airport'
]
def create_bucket_feature_cols() :
hour_buckets = tf.feature_column.bucketized_column(tf.feature_column.numeric_column('pickup_hour'), list(np.linspace(0, 23, 24)))
 weekday_buckets = tf.feature_column.bucketized_column(tf.feature_column.numeric_column('pickup_weekday'), list(np.linspace(0, 7, 8)))
 month_buckets = tf.feature_column.bucketized_column(tf.feature_column.numeric_column('pickup_month'), list(np.linspace(0, 11, 12)))
 year_buckets = tf.feature_column.bucketized_column(tf.feature_column.numeric_column('pickup_year'), list(np.linspace(2009, 2017, 9)))
 passenger_buckets = tf.feature_column.bucketized_column(tf.feature_column.numeric_column('passenger_count'), list(np.linspace(0, 8, 9)))
 distance_buckets = tf.feature_column.bucketized_column(tf.feature_column.numeric_column('distance'), list(np.linspace(0, 0.05, 100)))
 night_buckets = tf.feature_column.bucketized_column(tf.feature_column.numeric_column('night'), list(np.linspace(0, 1, 2)))
 angle_buckets = tf.feature_column.bucketized_column(tf.feature_column.numeric_column('angle'), list(np.linspace(-3.14, 3.14, 90)))
 NUM_BUCKETS = 27
long_list = list(np.linspace(-74.2, -73.7, NUM_BUCKETS))
lat_list = list(np.linspace(40.55, 41.0, NUM_BUCKETS))
p_lon = tf.feature_column.numeric_column('pickup_longitude')
 p_lat = tf.feature_column.numeric_column('pickup_latitude')
 d_lon = tf.feature_column.numeric_column('dropoff_longitude')
 d_lat = tf.feature_column.numeric_column('dropoff_latitude')
 buck_p_lon = tf.feature_column.bucketized_column(p_lon, long_list)
 buck_p_lat = tf.feature_column.bucketized_column(p_lat, lat_list)
 buck_d_lon = tf.feature_column.bucketized_column(d_lon, long_list)
 buck_d_lat = tf.feature_column.bucketized_column(d_lat, lat_list)
 return [
hour_buckets,
month_buckets,
year_buckets,
passenger_buckets,
distance_buckets,
angle_buckets,
night_buckets,
buck_p_lon,
buck_p_lat,
buck_d_lon,
buck_d_lat
] 
<statistical_test>
train = train.drop(["ID_code", "target"], axis=1 )
if DEBUG:
feat = create_bucket_feature_cols()
print("Number of features=", len(feat))
print(feat )<load_from_csv>
sns.set_style('whitegrid')
 sns.countplot(target)
 sns.set_style('whitegrid' )
BATCH_SIZE = 1024
train_input_fn = read_dataset(f'{PATH}/train.csv', tf.estimator.ModeKeys.TRAIN, batch_size = BATCH_SIZE)
 eval_input_fn = read_dataset(f'{PATH}/train.csv', tf.estimator.ModeKeys.EVAL, batch_size = BATCH_SIZE)
 def train_and_evaluate(output_dir, num_train_steps):
runconfig = tf.estimator.RunConfig(model_dir = OUTDIR, keep_checkpoint_max=1,
save_summary_steps=5000, log_step_count_steps=5000,
save_checkpoints_steps=10000,
tf_random_seed = 42 
)
 estimator = tf.estimator.BoostedTreesRegressor(model_dir = output_dir,
feature_columns = create_bucket_feature_cols() ,
n_batches_per_layer=128,
n_trees=400,
max_depth=6,
learning_rate=0.05,
l1_regularization=0.01,
l2_regularization=0.0,
tree_complexity=0.0,
min_node_weight=0.0,
center_bias = True,
config=runconfig)
 train_spec = tf.estimator.TrainSpec(input_fn = train_input_fn,
max_steps = num_train_steps)
 eval_spec = tf.estimator.EvalSpec(input_fn = eval_input_fn,
steps = None,
start_delay_secs = 0,
throttle_secs = 60)
 evaluation, result = tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
 return estimator, evaluation, result
OUTDIR = './trained_model'
shutil.rmtree(OUTDIR, ignore_errors = True)
 with timer('Train and Evaluate...'):
estimator, evaluation, result = train_and_evaluate(OUTDIR, 240000)
 print(evaluation )<save_to_csv>
@jit
def augment(x,y,t=2):
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xs.append(x1)
 for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xn.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn])
 y = np.concatenate([y,ys,yn])
 return x,y
print(evaluation)
 print(evaluation, file = sys.stderr)
 avg_loss = evaluation['average_loss']
predict_input_fn = read_dataset(f'{PATH}/test.csv', tf.estimator.ModeKeys.PREDICT, batch_size=1)
 predictions = estimator.predict(predict_input_fn)
 test_df = pd.read_csv(f'{PATH}/test.csv', nrows=10000)
 s = pd.Series()
for i, p in enumerate(predictions):
if i < 9915:
s.at[i] = p['predictions'][0]
else:
break
test_df['fare_amount'] = s
sub = test_df[['key', 'fare_amount']]
sub.to_csv(f'DNNregr-{avg_loss:4.4}.csv', index=False, float_format='%.1f')
<set_options>
param = {
'bagging_freq': 5,
'bagging_fraction': 0.335,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.041,
'learning_rate': 0.0083,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': -1
}
%matplotlib inline
SEED = 50
print(os.listdir(".. /input")) 
<load_from_csv>
num_folds = 11
features = [c for c in train.columns if c not in ['ID_code', 'target']]
folds = KFold(n_splits=num_folds, random_state=2319)
 oof = np.zeros(len(train))
getVal = np.zeros(len(train))
predictions = np.zeros(len(target))
feature_importance_df = pd.DataFrame()
dt_train = pd.read_csv(".. /input/train.csv")
 print("Number of Training Examples = {}".format(dt_train.shape))<load_from_csv>
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train.values, target.values)) :
X_train, y_train = train.iloc[trn_idx][features], target.iloc[trn_idx]
X_valid, y_valid = train.iloc[val_idx][features], target.iloc[val_idx]
X_tr, y_tr = augment(X_train.values, y_train.values)
 X_tr = pd.DataFrame(X_tr)
 print("Fold idx:{}".format(fold_ + 1))
trn_data = lgb.Dataset(X_tr, label=y_tr)
 val_data = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(param, trn_data, 1000000, valid_sets = [trn_data, val_data], verbose_eval=5000, early_stopping_rounds = 4000)
 oof[val_idx] = clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 getVal[val_idx]+= clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)/ folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test[features], num_iteration=clf.best_iteration)/ folds.n_splits
dt_test = pd.read_csv(".. /input/test.csv")
 print("Number of Test Examples = {}".format(dt_test.shape))<train_model>
print("
>> CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )
dt_train.name = "Training Dataset"
dt_test.name = "Test Dataset"
for dt in [dt_train, dt_test]:
print('{} have {} missing values.'.format(dt.name, int(dt.isnull().values.any())) )<count_values>
submission = pd.DataFrame({"ID_code": test.ID_code.values})
 submission["target"] = predictions
submission.to_csv("submission.csv", index=False )