Kaggle competition: New York City Taxi Fare Prediction
<SOS> metric: RMSE Kaggle data source: new-york-city-taxi-fare-prediction<train_model>
print(tf.__version__)
 assert tf.__version__ >= "1.8" or tf.__version__ >= "1.10"
tf.logging.set_verbosity(tf.logging.INFO)
 CSV_COLUMNS = ['fare_amount', 'pickup_datetime','pickup_longitude','pickup_latitude',
'dropoff_longitude','dropoff_latitude', 'passenger_count', 'key']
LABEL_COLUMN = 'fare_amount'
TRAIN_LINES = 55423856
DEBUG=False
fit = get_stacking().fit(X_train, Y_train )<predict_on_test>
@contextmanager
def timer(name):
t0 = time.time()
yield
print(f'[{name}] done in {time.time() - t0:.0f} s' )
test_sur = pd.Series(fit.predict(X_test), name ='Survived')
 test_sur<drop_column>
PATH = '.. /input'
train_df = pd.read_csv(f'{PATH}/train.csv', nrows=100000)
 train_df['distance'] = np.sqrt(np.abs(train_df['pickup_longitude']-train_df['dropoff_longitude'])**2 +
np.abs(train_df['pickup_latitude']-train_df['dropoff_latitude'])**2)
 train_df.head()
train_df.describe()
test1 = test[['PassengerId','Survived']]<data_type_conversions>
BATCH_SIZE=1
dataset = tf.contrib.data.make_csv_dataset(
 file_pattern=f'{PATH}/train.csv',
batch_size=BATCH_SIZE,
column_names=None,
column_defaults=None,
label_name='fare_amount',
select_columns=[1, 2, 3, 4, 5, 6, 7],
field_delim=',',
use_quote_delim=True,
na_value='',
header=True,
num_epochs=None,
shuffle=True,
shuffle_buffer_size=10000,
shuffle_seed=None,
prefetch_buffer_size=1,
num_parallel_reads=1,
sloppy=False,
num_rows_for_inference=100 
)
 if DEBUG:
next_element = dataset.make_one_shot_iterator().get_next()
with tf.Session() as sess:
features, label = sess.run(next_element)
 print("Features:
", features, "
Label:
", label)
test1['Survived'] = test_sur.values.astype(int)
 test1<save_to_csv>
def pd_weekDay(year, month, day):
df = pd.DataFrame({'year': year,
'month': month,
'day': day})
 date_df = pd.to_datetime(df)
 return date_df.dt.weekday.astype(np.int32)
 def pd_dayofYear(year, month, day):
df = pd.DataFrame({'year': year,
'month': month,
'day': day})
 date_df = pd.to_datetime(df)
 return date_df.dt.dayofyear.astype(np.int32)
 if DEBUG:
years=np.array([2018, 2018, 2018])
 months=np.array([8, 11, 1])
 days=np.array([20, 6, 8])
 print(pd_dayofYear(years, months, days))
test1.to_csv('Stacked.csv', index = False )<load_from_csv>
def tf_isAirport(latitude,longitude,airport_name='JFK'):
jfkcoord = tf.constant([-73.8352, -73.7401, 40.6195, 40.6659])
 ewrcoord = tf.constant([-74.1925, -74.1531, 40.6700, 40.7081])
 lgucoord = tf.constant([-73.8895, -73.8550, 40.7664, 40.7931])
 if airport_name=='JFK':
coord = jfkcoord
elif airport_name=='EWR':
coord = ewrcoord
elif airport_name=='LGU':
coord = lgucoord
else:
raise ValueError(f'Unknown NYC Airport {airport_name}')
 is_airport = \
tf.logical_and(
 tf.logical_and(
 tf.greater(longitude, coord[0]), tf.less(longitude, coord[1])
),
tf.logical_and(
 tf.greater(latitude, coord[2]), tf.less(latitude, coord[3])
)
)
 return is_airport
style.use('https://raw.githubusercontent.com/JoseGuzman/minibrain/master/minibrain/paper.mplstyle')
 train = pd.read_csv(".. /input/titanic/train.csv", index_col='PassengerId')
 test = pd.read_csv('.. /input/titanic/test.csv', index_col='PassengerId' )<groupby>
def feat_eng_func(features, label=None):
print("Feature Engineered Label:", label)
 features['pickup_year'] = tf.string_to_number(tf.substr(features['pickup_datetime'], 0, 4), tf.int32)
 features['pickup_month'] = tf.string_to_number(tf.substr(features['pickup_datetime'], 5, 2), tf.int32)
 features['pickup_day'] = tf.string_to_number(tf.substr(features['pickup_datetime'], 8, 2), tf.int32)
 features['pickup_hour'] = tf.string_to_number(tf.substr(features['pickup_datetime'], 11, 2), tf.int32)
 features['pickup_weekday'] = tf.py_func(pd_weekDay,
[features['pickup_year'], features['pickup_month'], features['pickup_day']],
tf.int32,
stateful=False,
name='Weekday' 
)
 features['night'] = tf.cast(
 tf.logical_or(tf.greater(features['pickup_hour'], 19), tf.less(features['pickup_hour'], 7)) ,
tf.float32)
 minlat = tf.constant(38.0)
 maxlat = tf.constant(42.0)
 minlon = tf.constant(-76.0)
 maxlon = tf.constant(-72.0)
 features['pickup_longitude'] = tf.clip_by_value(features['pickup_longitude'], minlon, maxlon)
 features['pickup_latitude'] = tf.clip_by_value(features['pickup_latitude'], minlat, maxlat)
 features['dropoff_longitude'] = tf.clip_by_value(features['dropoff_longitude'], minlon, maxlon)
 features['dropoff_latitude'] = tf.clip_by_value(features['dropoff_latitude'], minlat, maxlat)
 minpass = tf.constant(1.0)
 maxpass = tf.constant(6.0)
 features['passenger_count'] = tf.clip_by_value(tf.cast(features['passenger_count'], tf.float32), minpass, maxpass)
 if label != None:
minfare = tf.constant(3.0)
 maxfare = tf.constant(300.0)
 label = tf.clip_by_value(label, minfare, maxfare)
 features['longitude_dist'] = tf.abs(features['pickup_longitude'] - features['dropoff_longitude'])
 features['latitude_dist'] = tf.abs(features['pickup_latitude'] - features['dropoff_latitude'])
 features['distance'] = tf.sqrt(features['longitude_dist']**2 + features['latitude_dist']**2)
 features['pick_angle'] = angle_from_nyc(features['pickup_longitude'],features['pickup_latitude'])
 features['drop_angle'] = angle_from_nyc(features['dropoff_longitude'],features['dropoff_latitude'])
 features['angle'] = features['pick_angle'] - features['drop_angle']
if label == None:
return features
return(features, label )
mydf = train.groupby('Survived' ).mean()
mydf<groupby>
def read_dataset(filename, mode, batch_size = 512):
def _input_fn() :
if mode == tf.estimator.ModeKeys.TRAIN:
num_epochs = None
shuffle = False
else:
num_epochs = 1
shuffle = False
if mode == tf.estimator.ModeKeys.PREDICT:
label_name=None
select_columns=[1, 2, 3, 4, 5, 6]
else:
label_name ='fare_amount'
select_columns = [1, 2, 3, 4, 5, 6, 7]
file_list = tf.gfile.Glob(filename)
 dataset = tf.contrib.data.make_csv_dataset(
 file_pattern=file_list,
batch_size=batch_size,
column_names=None,
column_defaults=None,
label_name=label_name,
select_columns=select_columns,
field_delim=',',
use_quote_delim=True,
na_value='',
header=True,
num_epochs=num_epochs,
shuffle=shuffle,
shuffle_buffer_size=128*batch_size,
shuffle_seed=None,
prefetch_buffer_size=1,
num_parallel_reads=1,
sloppy=False,
num_rows_for_inference=100 
)
 skip_train_lines = TRAIN_LINES // batch_size // 100 * 10
if mode == tf.estimator.ModeKeys.TRAIN:
dataset = dataset.skip(skip_train_lines)
 elif mode == tf.estimator.ModeKeys.EVAL:
dataset = dataset.take(skip_train_lines)
 dataset = dataset.map(feat_eng_func)
 return dataset.make_one_shot_iterator().get_next()
return _input_fn
train.groupby('Sex')['Survived'].mean()<import_modules>
if DEBUG:
train_input_fn = read_dataset(f'{PATH}/train.csv', tf.estimator.ModeKeys.EVAL, batch_size = 8)
 with timer('Debugging'):
with tf.Session() as sess:
features, label = sess.run(train_input_fn())
 print("Features:
", features, "
Label:
", label )
from sklearn.pipeline import Pipeline<train_model>
NUMERIC_COLUMNS = ['passenger_count', 'pickup_dense_year',
'longitude_dist', 'latitude_dist',
'distance', 'pick_dist_center', 'drop_dist_center', 'angle',
'pick_angle','drop_angle'
]
BOOL_COLUMNS = ['is_JFK_pickup', 'is_JFK_dropoff', 'is_EWR_pickup', 'is_EWR_dropoff',
'is_LGU_pickup', 'is_LGU_dropoff', 'is_NYC_airport'
]
def create_bucket_feature_cols() :
hour_buckets = tf.feature_column.bucketized_column(tf.feature_column.numeric_column('pickup_hour'), list(np.linspace(0, 23, 24)))
 weekday_buckets = tf.feature_column.bucketized_column(tf.feature_column.numeric_column('pickup_weekday'), list(np.linspace(0, 7, 8)))
 month_buckets = tf.feature_column.bucketized_column(tf.feature_column.numeric_column('pickup_month'), list(np.linspace(0, 11, 12)))
 year_buckets = tf.feature_column.bucketized_column(tf.feature_column.numeric_column('pickup_year'), list(np.linspace(2009, 2017, 9)))
 passenger_buckets = tf.feature_column.bucketized_column(tf.feature_column.numeric_column('passenger_count'), list(np.linspace(0, 8, 9)))
 distance_buckets = tf.feature_column.bucketized_column(tf.feature_column.numeric_column('distance'), list(np.linspace(0, 0.05, 100)))
 night_buckets = tf.feature_column.bucketized_column(tf.feature_column.numeric_column('night'), list(np.linspace(0, 1, 2)))
 angle_buckets = tf.feature_column.bucketized_column(tf.feature_column.numeric_column('angle'), list(np.linspace(-3.14, 3.14, 90)))
 NUM_BUCKETS = 27
long_list = list(np.linspace(-74.2, -73.7, NUM_BUCKETS))
lat_list = list(np.linspace(40.55, 41.0, NUM_BUCKETS))
p_lon = tf.feature_column.numeric_column('pickup_longitude')
 p_lat = tf.feature_column.numeric_column('pickup_latitude')
 d_lon = tf.feature_column.numeric_column('dropoff_longitude')
 d_lat = tf.feature_column.numeric_column('dropoff_latitude')
 buck_p_lon = tf.feature_column.bucketized_column(p_lon, long_list)
 buck_p_lat = tf.feature_column.bucketized_column(p_lat, lat_list)
 buck_d_lon = tf.feature_column.bucketized_column(d_lon, long_list)
 buck_d_lat = tf.feature_column.bucketized_column(d_lat, lat_list)
 return [
hour_buckets,
month_buckets,
year_buckets,
passenger_buckets,
distance_buckets,
angle_buckets,
night_buckets,
buck_p_lon,
buck_p_lat,
buck_d_lon,
buck_d_lat
]
class FeaturesTransformer() :

 def __init__(self, create_family=True, dissect_cabin = True):

 self.create_family = create_family
self.dissect_cabin = dissect_cabin
def fit(self, X, y=None, **fit_params):

 return self
def transform(self, X, **transform_params):

 mydf = X.copy()
mydf['Title'] = X.Name.str.extract('([A-Za-z]+)\.', expand=False)
 mydf.drop(['Name'], axis = 1, inplace = True)
 mydf['Title'].replace(['Ms','Lady', 'Countess','Dona'],'Mrs', inplace=True)
 mydf['Title'].replace(['Mme','Mlle'], 'Miss', inplace = True)
 mydf['Title'].replace(['Major', 'Sir', 'Jonkheer', 'Dr','Col','Don', 'Capt','Rev'],'Mr', inplace=True)
 if self.create_family:
mydf['FamilySize'] = X['SibSp'] + X['Parch'] + 1
mydf.drop(['SibSp', 'Parch'], axis=1, inplace=True)
 if self.dissect_cabin:
mydf['Cabin'].fillna('N', inplace=True)
 mydf['Room'] = X['Cabin'].str.extract("([0-9]+)", expand=False ).astype("float")
 mydf['nRooms'] = mydf.Cabin.apply(lambda x: len(str(x ).split()))
mydf.loc[mydf.Cabin == 'N', 'nRooms'] = 0
mydf['CabinDeck'] = mydf.Cabin.apply(lambda x: str(x[0]))
mydf['CabinDeck'].replace(['T'], 'A', inplace=True)
 del mydf['Cabin']
return mydf<categorify>
if DEBUG:
feat = create_bucket_feature_cols()
print("Number of features=", len(feat))
print(feat )
preprocess = Pipeline([ 
('Preprocessing', FeaturesTransformer(create_family=True))
])
 mytrain = preprocess.fit_transform(train )<groupby>
BATCH_SIZE = 1024
train_input_fn = read_dataset(f'{PATH}/train.csv', tf.estimator.ModeKeys.TRAIN, batch_size = BATCH_SIZE)
 eval_input_fn = read_dataset(f'{PATH}/train.csv', tf.estimator.ModeKeys.EVAL, batch_size = BATCH_SIZE)
 def train_and_evaluate(output_dir, num_train_steps):
runconfig = tf.estimator.RunConfig(model_dir = OUTDIR, keep_checkpoint_max=1,
save_summary_steps=5000, log_step_count_steps=5000,
save_checkpoints_steps=10000,
tf_random_seed = 42 
)
 estimator = tf.estimator.BoostedTreesRegressor(model_dir = output_dir,
feature_columns = create_bucket_feature_cols() ,
n_batches_per_layer=128,
n_trees=400,
max_depth=6,
learning_rate=0.05,
l1_regularization=0.01,
l2_regularization=0.0,
tree_complexity=0.0,
min_node_weight=0.0,
center_bias = True,
config=runconfig)
 train_spec = tf.estimator.TrainSpec(input_fn = train_input_fn,
max_steps = num_train_steps)
 eval_spec = tf.estimator.EvalSpec(input_fn = eval_input_fn,
steps = None,
start_delay_secs = 0,
throttle_secs = 60)
 evaluation, result = tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
 return estimator, evaluation, result
OUTDIR = './trained_model'
shutil.rmtree(OUTDIR, ignore_errors = True)
 with timer('Train and Evaluate...'):
estimator, evaluation, result = train_and_evaluate(OUTDIR, 240000)
 print(evaluation )
mytrain.groupby('Survived')['Title'].value_counts()<categorify>
print(evaluation)
 print(evaluation, file = sys.stderr)
 avg_loss = evaluation['average_loss']
predict_input_fn = read_dataset(f'{PATH}/test.csv', tf.estimator.ModeKeys.PREDICT, batch_size=1)
 predictions = estimator.predict(predict_input_fn)
 test_df = pd.read_csv(f'{PATH}/test.csv', nrows=10000)
 s = pd.Series()
for i, p in enumerate(predictions):
if i < 9915:
s.at[i] = p['predictions'][0]
else:
break
test_df['fare_amount'] = s
sub = test_df[['key', 'fare_amount']]
sub.to_csv(f'DNNregr-{avg_loss:4.4}.csv', index=False, float_format='%.1f')