Kaggle competition: New York City Taxi Fare Prediction
<SOS> metric: RMSE Kaggle data source: new-york-city-taxi-fare-prediction<load_from_csv>
os.environ['OMP_NUM_THREADS'] = '4'
os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
print(tf.__version__)
 assert tf.__version__ >= "1.8" or tf.__version__ >= "1.10"
tf.logging.set_verbosity(tf.logging.INFO)
 CSV_COLUMNS = ['fare_amount', 'pickup_datetime','pickup_longitude','pickup_latitude',
'dropoff_longitude','dropoff_latitude', 'passenger_count', 'key']
LABEL_COLUMN = 'fare_amount'
TRAIN_LINES = 55423856
DEBUG=False
train_df = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 test_df = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 train_df.head()<count_missing_values>
@contextmanager
def timer(name):
t0 = time.time()
yield
print(f'[{name}] done in {time.time() - t0:.0f} s' )
print(train_df[train_df['text'].isnull() ])
 print(train_df[train_df['selected_text'].isnull() ] )<drop_column>
PATH = '.. /input'
train_df = pd.read_csv(f'{PATH}/train.csv', nrows=10000)
 train_df['distance'] = np.sqrt(np.abs(train_df['pickup_longitude']-train_df['dropoff_longitude'])**2 +
np.abs(train_df['pickup_latitude']-train_df['dropoff_latitude'])**2)
 train_df.head()
train_df.describe()
train_df.dropna(inplace = True)
 train_df = train_df.reset_index(drop = True )<feature_engineering>
BATCH_SIZE=1
dataset = tf.contrib.data.make_csv_dataset(
 file_pattern=f'{PATH}/train.csv',
batch_size=BATCH_SIZE,
column_names=None,
column_defaults=None,
label_name='fare_amount',
select_columns=[1, 2, 3, 4, 5, 6, 7],
field_delim=',',
use_quote_delim=True,
na_value='',
header=True,
num_epochs=None,
shuffle=True,
shuffle_buffer_size=10000,
shuffle_seed=None,
prefetch_buffer_size=1,
num_parallel_reads=1,
sloppy=False,
num_rows_for_inference=100
)
train_df['N_text_words'] = train_df['text'].apply(lambda tweet : len(tweet.split()))
train_df['N_selected_text_words'] = train_df['selected_text'].apply(lambda tweet : len(tweet.split()))
train_df['N_words_difference'] = train_df['N_text_words'] - train_df['N_selected_text_words']<count_unique_values>
def filter_data(features, label=None):
val = tf.greater(label , tf.constant(20.0, tf.float32))
return True
if DEBUG:
dataset = dataset.filter(filter_data)
 next_element = dataset.make_one_shot_iterator().get_next()
with tf.Session() as sess:
features, label = sess.run(next_element)
 print("Features:
", features, "
Label:
", label)
print("There are {0} unique sentiments having values {1}".format(train_df['sentiment'].nunique() , train_df['sentiment'].unique()))<count_values>
def pd_weekDay(year, month, day):
df = pd.DataFrame({'year': year,
'month': month,
'day': day})
 date_df = pd.to_datetime(df)
 return date_df.dt.weekday.astype(np.int32)
 def pd_dayofYear(year, month, day):
df = pd.DataFrame({'year': year,
'month': month,
'day': day})
 date_df = pd.to_datetime(df)
 return date_df.dt.dayofyear.astype(np.int32)
 if DEBUG:
years=np.array([2018, 2018, 2018])
 months=np.array([8, 11, 1])
 days=np.array([20, 6, 8])
 print(pd_dayofYear(years, months, days))
n_neutral = train_df['sentiment'].loc[train_df['sentiment'] == 'neutral'].count()
n_positive = train_df['sentiment'].loc[train_df['sentiment'] == 'positive'].count()
n_negative = train_df['sentiment'].loc[train_df['sentiment'] == 'negative'].count()<train_model>
def tf_isAirport(latitude,longitude,airport_name='JFK'):
jfkcoord = tf.constant([-73.8352, -73.7401, 40.6195, 40.6659])
 ewrcoord = tf.constant([-74.1925, -74.1531, 40.6700, 40.7081])
 lgucoord = tf.constant([-73.8895, -73.8550, 40.7664, 40.7931])
 if airport_name=='JFK':
coord = jfkcoord
elif airport_name=='EWR':
coord = ewrcoord
elif airport_name=='LGU':
coord = lgucoord
else:
raise ValueError(f'Unknown NYC Airport {airport_name}')
 is_airport = \
tf.logical_and(
 tf.logical_and(
 tf.greater(longitude, coord[0]), tf.less(longitude, coord[1])
),
tf.logical_and(
 tf.greater(latitude, coord[2]), tf.less(latitude, coord[3])
)
)
 return is_airport
print(f"Neutral tweets : {n_neutral}")
 print(f"Positive tweets : {n_positive}")
 print(f"Negative tweets : {n_negative}" )<filter>
def feat_eng_func(features, label=None):
print("Feature Engineered Label:", label)
 features['pickup_year'] = tf.string_to_number(tf.substr(features['pickup_datetime'], 0, 4), tf.int32)
 features['pickup_month'] = tf.string_to_number(tf.substr(features['pickup_datetime'], 5, 2), tf.int32)
 features['pickup_day'] = tf.string_to_number(tf.substr(features['pickup_datetime'], 8, 2), tf.int32)
 features['pickup_hour'] = tf.string_to_number(tf.substr(features['pickup_datetime'], 11, 2), tf.int32)
 features['pickup_weekday'] = tf.py_func(pd_weekDay,
[features['pickup_year'], features['pickup_month'], features['pickup_day']],
tf.int32,
stateful=False,
name='Weekday' 
)
 features['pickup_dense_year'] =(
 tf.cast(features['pickup_year'], tf.float32)+ \
tf.cast(features['pickup_month'], tf.float32)/ tf.constant(12.0, tf.float32)- \
tf.constant(2009.0, tf.float32)) / \
tf.constant(6.0, tf.float32)
 features['night'] = tf.cast(
 tf.logical_or(tf.greater(features['pickup_hour'], 19), tf.less(features['pickup_hour'], 7)) ,
tf.float32)
 minlat = tf.constant(38.0)
 maxlat = tf.constant(42.0)
 minlon = tf.constant(-76.0)
 maxlon = tf.constant(-72.0)
 features['pickup_longitude'] = tf.clip_by_value(features['pickup_longitude'], minlon, maxlon)
 features['pickup_latitude'] = tf.clip_by_value(features['pickup_latitude'], minlat, maxlat)
 features['dropoff_longitude'] = tf.clip_by_value(features['dropoff_longitude'], minlon, maxlon)
 features['dropoff_latitude'] = tf.clip_by_value(features['dropoff_latitude'], minlat, maxlat)
 minpass = tf.constant(1.0)
 maxpass = tf.constant(6.0)
 features['passenger_count'] = tf.clip_by_value(tf.cast(features['passenger_count'], tf.float32), minpass, maxpass)
 if label != None:
minfare = tf.constant(3.0)
 maxfare = tf.constant(300.0)
 label = tf.clip_by_value(label, minfare, maxfare)
 features['longitude_dist'] = tf.abs(features['pickup_longitude'] - features['dropoff_longitude'])
 features['latitude_dist'] = tf.abs(features['pickup_latitude'] - features['dropoff_latitude'])
 features['distance'] = tf.sqrt(features['longitude_dist']**2 + features['latitude_dist']**2)
 features['pick_dist_center'] = distance_from_loc(nyccenter[0], nyccenter[1],
features['pickup_longitude'], features['pickup_latitude'])
 features['drop_dist_center'] = distance_from_loc(nyccenter[0], nyccenter[1],
features['dropoff_longitude'], features['dropoff_latitude'])
 features['pick_angle'] = angle_from_nyc(features['pickup_longitude'],features['pickup_latitude'])
 features['drop_angle'] = angle_from_nyc(features['dropoff_longitude'],features['dropoff_latitude'])
 features['angle'] = features['pick_angle'] - features['drop_angle']
if label == None:
return features
return(features, label )
df_neutral = train_df.loc[train_df['sentiment'] == 'neutral']
df_positive = train_df.loc[train_df['sentiment'] == 'positive']
df_negative = train_df.loc[train_df['sentiment'] == 'negative']<string_transform>
def read_dataset(filename, mode, batch_size = 512):
def _input_fn() :
if mode == tf.estimator.ModeKeys.TRAIN:
num_epochs = None
shuffle = False
else:
num_epochs = 1
shuffle = False
if mode == tf.estimator.ModeKeys.PREDICT:
label_name=None
select_columns=[1, 2, 3, 4, 5, 6]
else:
label_name ='fare_amount'
select_columns = [1, 2, 3, 4, 5, 6, 7]
file_list = tf.gfile.Glob(filename)
 dataset = tf.contrib.data.make_csv_dataset(
 file_pattern=file_list,
batch_size=batch_size,
column_names=None,
column_defaults=None,
label_name=label_name,
select_columns=select_columns,
field_delim=',',
use_quote_delim=True,
na_value='',
header=True,
num_epochs=num_epochs,
shuffle=shuffle,
shuffle_buffer_size=128*batch_size,
shuffle_seed=None,
prefetch_buffer_size=1,
num_parallel_reads=1,
sloppy=False,
num_rows_for_inference=100 
)
 skip_train_lines = TRAIN_LINES // batch_size // 100 * 10
if mode == tf.estimator.ModeKeys.TRAIN:
dataset = dataset.skip(skip_train_lines)
 elif mode == tf.estimator.ModeKeys.EVAL:
dataset = dataset.take(skip_train_lines)
 dataset = dataset.map(feat_eng_func)
 return dataset.make_one_shot_iterator().get_next()
return _input_fn
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<prepare_output>
train_input_fn = read_dataset(f'{PATH}/train.csv', tf.estimator.ModeKeys.EVAL, batch_size = 8)
train_df['Jaccard_score'] = jaccard_score<categorify>
with timer('Evaluating'):
with tf.Session() as sess:
features, label = sess.run(train_input_fn())
 print("Features:
", features, "
Label:
", label )
def clean_text(text):

 text = str(text ).lower()
text = re.sub('\[.*?\]', '', text)
 text = re.sub('https?://\S+|www\.\S+', '', text)
 text = re.sub('<.*?>+', '', text)
 text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
 text = re.sub('
', '', text)
 text = re.sub('\w*\d\w*', '', text)
 return text<feature_engineering>
NUMERIC_COLUMNS = ['passenger_count', 'pickup_dense_year',
'longitude_dist', 'latitude_dist',
'distance', 'pick_dist_center', 'drop_dist_center', 'angle',
'pick_angle','drop_angle',
]
BOOL_COLUMNS = ['is_JFK_pickup', 'is_JFK_dropoff', 'is_EWR_pickup', 'is_EWR_dropoff',
'is_LGU_pickup', 'is_LGU_dropoff', 'is_NYC_airport'
]
def create_feature_cols() :
hour_cat = tf.feature_column.categorical_column_with_identity('pickup_hour', 24)
 weekday_cat = tf.feature_column.categorical_column_with_identity('pickup_weekday', 7)
 hour_X_weekday = tf.feature_column.crossed_column([hour_cat, weekday_cat], 500)
 NUM_BUCKETS = 27
long_list = list(np.linspace(-74.2, -73.7, NUM_BUCKETS))
lat_list = list(np.linspace(40.55, 41.0, NUM_BUCKETS))
p_lon = tf.feature_column.numeric_column('pickup_longitude')
 p_lat = tf.feature_column.numeric_column('pickup_latitude')
 d_lon = tf.feature_column.numeric_column('dropoff_longitude')
 d_lat = tf.feature_column.numeric_column('dropoff_latitude')
 buck_p_lon = tf.feature_column.bucketized_column(p_lon, long_list)
 buck_p_lat = tf.feature_column.bucketized_column(p_lat, lat_list)
 buck_d_lon = tf.feature_column.bucketized_column(d_lon, long_list)
 buck_d_lat = tf.feature_column.bucketized_column(d_lat, lat_list)
 X_p = tf.feature_column.crossed_column([buck_p_lon, buck_p_lat], 3*NUM_BUCKETS**2)
 X_d = tf.feature_column.crossed_column([buck_d_lon, buck_d_lat], 3*NUM_BUCKETS**2)
 X_loc = tf.feature_column.crossed_column([X_p, X_d],(3*NUM_BUCKETS**2)**2)
 numeric_columns = [ tf.feature_column.numeric_column(key)
 for key in NUMERIC_COLUMNS]
other_columns = [tf.feature_column.indicator_column(hour_cat),
tf.feature_column.embedding_column(hour_X_weekday, 2),
tf.feature_column.embedding_column(
 tf.feature_column.categorical_column_with_vocabulary_list('pickup_month',(0, 1, 2, 3, 4, 5, 6,
7, 8, 9, 10, 11, 12)) ,
2),
tf.feature_column.embedding_column(X_loc, NUM_BUCKETS)
 ]
return numeric_columns + other_columns
def create_sparse_feature_cols() :
bool_columns = [ tf.feature_column.indicator_column(
 tf.feature_column.categorical_column_with_identity(key, 2))
for key in BOOL_COLUMNS]
return bool_columns
train_df['text_cleaned'] = train_df['text'].apply(lambda x:clean_text(x))
train_df['selected_text_cleaned'] = train_df['selected_text'].apply(lambda x:clean_text(x))<string_transform>
BATCH_SIZE = 512
train_input_fn = read_dataset(f'{PATH}/train.csv', tf.estimator.ModeKeys.TRAIN, batch_size = BATCH_SIZE)
 eval_input_fn = read_dataset(f'{PATH}/train.csv', tf.estimator.ModeKeys.EVAL, batch_size = BATCH_SIZE)
 def train_and_evaluate(output_dir, num_train_steps):
runconfig = tf.estimator.RunConfig(model_dir = OUTDIR, keep_checkpoint_max=1,
save_summary_steps=25000, log_step_count_steps=25000,
save_checkpoints_steps=50000,
tf_random_seed = 42 
)
 optimizer = tf.contrib.opt.LazyAdamOptimizer(
)
 estimator = tf.estimator.DNNRegressor(model_dir = output_dir, feature_columns = create_feature_cols() ,
hidden_units=[128, 128, 128],
optimizer=optimizer,
config=runconfig,
batch_norm=True,
loss_reduction = tf.losses.Reduction.SUM_OVER_BATCH_SIZE)
 train_spec = tf.estimator.TrainSpec(input_fn = train_input_fn,
max_steps = num_train_steps)
 eval_spec = tf.estimator.EvalSpec(input_fn = eval_input_fn,
steps = None,
start_delay_secs = 0,
throttle_secs = 60)
 evaluation, result = tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
 return estimator, evaluation, result
OUTDIR = './trained_model'
shutil.rmtree(OUTDIR, ignore_errors = True)
 estimator, evaluation, result = train_and_evaluate(OUTDIR, 180000 )
STOPWORDS = stopwords.words('english')
 def remove_stopwords(text):
return [word for word in text.split() if word not in STOPWORDS]<feature_engineering>
if False:
BATCH_SIZE = 512
OUTDIR = './trained_model'
train_input_fn = read_dataset(f'{PATH}/train.csv', tf.estimator.ModeKeys.TRAIN, batch_size = BATCH_SIZE)
 eval_input_fn = read_dataset(f'{PATH}/train.csv', tf.estimator.ModeKeys.EVAL, batch_size = BATCH_SIZE)
 shutil.rmtree(OUTDIR, ignore_errors = True)
 runconfig = tf.estimator.RunConfig(model_dir = OUTDIR, keep_checkpoint_max=1,
save_summary_steps=2500, log_step_count_steps=2500,
save_checkpoints_steps=500000,
tf_random_seed = 42 
)
 optimizer = tf.train.FtrlOptimizer(learning_rate=0.05,
learning_rate_power=-0.5,
initial_accumulator_value=0.1,
l1_regularization_strength=0.0,
l2_regularization_strength=0.0 
)
 estimator = tf.estimator.DNNRegressor(model_dir = OUTDIR, feature_columns = create_feature_cols() ,
hidden_units=[128, 128, 128],
batch_norm=True,
config=runconfig,
dropout=0.0)
 with timer('Training...'):
estimator.train(train_input_fn, max_steps=130000)
 with timer('Evaluating'):
evaluation = estimator.evaluate(eval_input_fn, name='train_eval')
train_df['text_cleaned'] = train_df['text_cleaned'].apply(lambda x : remove_stopwords(x))
train_df['selected_text_cleaned'] = train_df['selected_text_cleaned'].apply(lambda x : remove_stopwords(x))<string_transform>
print(evaluation)
 avg_loss = evaluation['average_loss']
predict_input_fn = read_dataset(f'{PATH}/test.csv', tf.estimator.ModeKeys.PREDICT, batch_size=1)
 predictions = estimator.predict(predict_input_fn)
 test_df = pd.read_csv(f'{PATH}/test.csv', nrows=10000)
 s = pd.Series()
for i, p in enumerate(predictions):
if i < 9915:
s.at[i] = p['predictions'][0]
else:
break
test_df['fare_amount'] = s
sub = test_df[['key', 'fare_amount']]
sub.to_csv(f'DNNregr-{avg_loss:4.4}.csv', index=False, float_format='%.1f')