Kaggle competition: New York City Taxi Fare Prediction
<SOS> metric: RMSE Kaggle data source: new-york-city-taxi-fare-prediction<load_from_disk>
import numpy as np
import pandas as pd
from datetime import datetime
import matplotlib.pyplot as plt
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from keras.models import Sequential
from keras.layers import Dense, Dropout, BatchNormalization
from keras.callbacks import EarlyStopping
from keras import optimizers
from keras import regularizers
def get_data(task_filename):
with open(task_filename, 'r')as f:
task = json.load(f)
 return task
num2color = ["black", "blue", "red", "green", "yellow", "gray", "magenta", "orange", "sky", "brown"]
color2num = {c: n for n, c in enumerate(num2color)}<define_variables>
def clean(df):
df = df[(-76 <= df['pickup_longitude'])&(df['pickup_longitude'] <= -72)]
df = df[(-76 <= df['dropoff_longitude'])&(df['dropoff_longitude'] <= -72)]
df = df[(38 <= df['pickup_latitude'])&(df['pickup_latitude'] <= 42)]
df = df[(38 <= df['dropoff_latitude'])&(df['dropoff_latitude'] <= 42)]
df = df[(0 < df['fare_amount'])&(df['fare_amount'] <= 250)]
df = df[(df['dropoff_longitude'] != df['pickup_longitude'])]
df = df[(df['dropoff_latitude'] != df['pickup_latitude'])]
return df
for i in range(400):
print(get_data(str(test_path / test_tasks[i])))
 break<define_variables>
def late_night(row):
if(row['hour'] <= 6)or(row['hour'] >= 20):
return 1
else:
return 0
def night(row):
if(( row['hour'] <= 20)and(row['hour'] >= 16)) and(row['weekday'] < 5):
return 1
else:
return 0
def manhattan(pickup_lat, pickup_long, dropoff_lat, dropoff_long):
return np.abs(dropoff_lat - pickup_lat)+ np.abs(dropoff_long - pickup_long)
 def add_time_features(df):
df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'], format='%Y-%m-%d %H:%M:%S %Z')
 df['year'] = df['pickup_datetime'].apply(lambda x: x.year)
 df['month'] = df['pickup_datetime'].apply(lambda x: x.month)
 df['day'] = df['pickup_datetime'].apply(lambda x: x.day)
 df['hour'] = df['pickup_datetime'].apply(lambda x: x.hour)
 df['weekday'] = df['pickup_datetime'].apply(lambda x: x.weekday())
 df['pickup_datetime'] = df['pickup_datetime'].apply(lambda x: str(x))
df['night'] = df.apply(lambda x: night(x), axis=1)
 df['late_night'] = df.apply(lambda x: late_night(x), axis=1)
 df = df.drop('pickup_datetime', axis=1)
 return df
def add_coordinate_features(df):
lat1 = df['pickup_latitude']
lat2 = df['dropoff_latitude']
lon1 = df['pickup_longitude']
lon2 = df['dropoff_longitude']
df['latdiff'] =(lat1 - lat2)
 df['londiff'] =(lon1 - lon2)
 return df
def add_distances_features(df):
ny =(-74.0063889, 40.7141667)
 jfk =(-73.7822222222, 40.6441666667)
 ewr =(-74.175, 40.69)
 lgr =(-73.87, 40.77)
 lat1 = df['pickup_latitude']
lat2 = df['dropoff_latitude']
lon1 = df['pickup_longitude']
lon2 = df['dropoff_longitude']
df['euclidean'] =(df['latdiff'] ** 2 + df['londiff'] ** 2)** 0.5
df['manhattan'] = manhattan(lat1, lon1, lat2, lon2)
 df['downtown_pickup_distance'] = manhattan(ny[1], ny[0], lat1, lon1)
 df['downtown_dropoff_distance'] = manhattan(ny[1], ny[0], lat2, lon2)
 df['jfk_pickup_distance'] = manhattan(jfk[1], jfk[0], lat1, lon1)
 df['jfk_dropoff_distance'] = manhattan(jfk[1], jfk[0], lat2, lon2)
 df['ewr_pickup_distance'] = manhattan(ewr[1], ewr[0], lat1, lon1)
 df['ewr_dropoff_distance'] = manhattan(ewr[1], ewr[0], lat2, lon2)
 df['lgr_pickup_distance'] = manhattan(lgr[1], lgr[0], lat1, lon1)
 df['lgr_dropoff_distance'] = manhattan(lgr[1], lgr[0], lat2, lon2)
 return df
for i in range(400):
print(get_data(str(training_path / training_tasks[i])) ['test'])
 break<prepare_x_and_y>
def output_submission(raw_test, prediction, id_column, prediction_column, file_name):
df = pd.DataFrame(prediction, columns=[prediction_column])
 df[id_column] = raw_test[id_column]
df[[id_column, prediction_column]].to_csv(( file_name), index=False)
 print('Output complete')
 def plot_loss_accuracy(history):
plt.figure(figsize=(20,10))
plt.plot(history.history['loss'])
 plt.plot(history.history['val_loss'])
 plt.title('model loss')
 plt.ylabel('loss')
 plt.xlabel('epoch')
 plt.legend(['train', 'test'], loc='upper right')
 plt.show()
x_train = []
y_train = []
x_test = []
y_test = []
ox_train = []
oy_train = []
ox_test = []
for i in range(400):
for train_data in get_data(str(training_path / training_tasks[i])) ['train']:
x_train.append(cv2.resize(np.asarray(train_data['input']), dsize=(32, 32), interpolation=cv2.INTER_NEAREST))
y_train.append(cv2.resize(np.asarray(train_data['output']), dsize=(32, 32), interpolation=cv2.INTER_NEAREST))
ox_train.append(np.asarray(train_data['input']))
oy_train.append(np.asarray(train_data['output']))
for i in range(100):
for test_data in get_data(str(test_path / test_tasks[i])) ['test']:
x_test.append(cv2.resize(np.asarray(test_data['input']), dsize=(32, 32), interpolation=cv2.INTER_NEAREST))
ox_test.append(np.asarray(test_data['input']))
for train_data in get_data(str(test_path / test_tasks[i])) ['train']:
x_train.append(cv2.resize(np.asarray(train_data['input']), dsize=(32, 32), interpolation=cv2.INTER_NEAREST))
y_train.append(cv2.resize(np.asarray(train_data['output']), dsize=(32, 32), interpolation=cv2.INTER_NEAREST))
ox_train.append(np.asarray(train_data['input']))
oy_train.append(np.asarray(train_data['output']))
x_train = np.asarray(x_train)/ 10.
 y_train = np.asarray(y_train)/ 10.
 x_test = np.asarray(x_test)/ 10.
 print('length of x_train:', len(x_train))
print('length of x_test:', len(x_test))<randomize_order>
TRAIN_PATH = '.. /input/train.csv'
TEST_PATH = '.. /input/test.csv'
SUBMISSION_NAME = 'submission.csv'
BATCH_SIZE = 256
EPOCHS = 50
LEARNING_RATE = 0.001
DATASET_SIZE = 6000000
def get_random_eraser(p=0.5, s_l=0.02, s_h=0.4, r_1=0.3, r_2=1/0.3, v_l=0, v_h=255, pixel_level=False):
def eraser(input_img):
img_h, img_w, img_c = input_img.shape
p_1 = np.random.rand()
if p_1 > p:
return input_img
while True:
s = np.random.uniform(s_l, s_h)* img_h * img_w
r = np.random.uniform(r_1, r_2)
 w = int(np.sqrt(s / r))
h = int(np.sqrt(s * r))
left = np.random.randint(0, img_w)
 top = np.random.randint(0, img_h)
 if left + w <= img_w and top + h <= img_h:
break
if pixel_level:
c = np.random.uniform(v_l, v_h,(h, w, img_c))
else:
c = np.random.uniform(v_l, v_h)
 input_img[top:top + h, left:left + w, :] = c
return input_img
return eraser<train_model>
datatypes = {'key': 'str',
'fare_amount': 'float32',
'pickup_datetime': 'str',
'pickup_longitude': 'float32',
'pickup_latitude': 'float32',
'dropoff_longitude': 'float32',
'dropoff_latitude': 'float32',
'passenger_count': 'uint8'}
train = pd.read_csv(TRAIN_PATH, nrows=DATASET_SIZE, dtype=datatypes, usecols=[1,2,3,4,5,6])
 test = pd.read_csv(TEST_PATH )
datagen = ImageDataGenerator(
 width_shift_range=0.5,
height_shift_range=0.5,
dtype=float,
fill_mode='nearest',
preprocessing_function = get_random_eraser(p=0.8, s_l=0.0009765625, s_h=0.0009765625, r_1=0.03124, r_2=0.03124, v_l=0, v_h=9, pixel_level=True), 
)
 x_train = x_train.reshape(x_train.shape +(1,))
datagen.fit(x_train)
 y_train = y_train.reshape(y_train.shape +(1,))
x_test = x_test.reshape(x_test.shape +(1,))
datagen.fit(x_test )<define_variables>
train = clean(train)
 train = add_time_features(train)
 test = add_time_features(test)
 add_coordinate_features(train)
 add_coordinate_features(test)
 train = add_distances_features(train)
 test = add_distances_features(test)
 train.head(5 )
def sampling(args):
z_mean, z_log_var = args
batch = K.shape(z_mean)[0]
dim = K.int_shape(z_mean)[1]
epsilon = K.random_normal(shape=(batch, dim))
return z_mean + K.exp(0.5 * z_log_var)* epsilon<define_variables>
dropped_columns = ['pickup_longitude', 'pickup_latitude',
'dropoff_longitude', 'dropoff_latitude']
train_clean = train.drop(dropped_columns, axis=1)
 test_clean = test.drop(dropped_columns + ['key', 'passenger_count'], axis=1)
 train_clean.head(5 )
original_dim = 32 * 32
input_shape_vae =(original_dim, )<define_variables>
train_df, validation_df = train_test_split(train_clean, test_size=0.10, random_state=1)
 train_labels = train_df['fare_amount'].values
validation_labels = validation_df['fare_amount'].values
train_df = train_df.drop(['fare_amount'], axis=1)
 validation_df = validation_df.drop(['fare_amount'], axis=1 )
intermediate_dim = 512
batch_size = 32
latent_dim = 256
epochs = 100<categorify>
scaler = preprocessing.MinMaxScaler()
train_df_scaled = scaler.fit_transform(train_df)
 validation_df_scaled = scaler.transform(validation_df)
 test_scaled = scaler.transform(test_clean )
inputs = Input(shape=input_shape_vae)
 x = Dense(intermediate_dim, activation='relu' )(inputs)
 x = Dropout(0.5 )(x)
 x = Dense(intermediate_dim, activation='relu' )(x)
 x = Dropout(0.3 )(x)
 x = Dense(intermediate_dim, activation='relu' )(x)
 z_mean = Dense(latent_dim )(x)
 z_log_var = Dense(latent_dim )(x)
 z = Lambda(sampling, output_shape=(latent_dim,))([z_mean, z_log_var])
 encoder = Model(inputs, [z_mean, z_log_var, z], name='encoder')
 encoder.summary()
latent_inputs = Input(shape=(latent_dim,), name='z_sampling')
 x = Dense(intermediate_dim, activation='relu' )(latent_inputs)
 x = Dropout(0.3 )(x)
 x = Dense(intermediate_dim, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 x = Dense(intermediate_dim, activation='relu' )(x)
 outputs = Dense(original_dim, activation='sigmoid' )(x)
 decoder = Model(latent_inputs, outputs, name='decoder')
 decoder.summary()
outputs = decoder(encoder(inputs)[2])
 vae = Model(inputs, outputs, name='vae_mlp' )<train_model>
model = Sequential()
model.add(Dense(256, activation='relu', input_dim=train_df_scaled.shape[1], activity_regularizer=regularizers.l1(0.01)))
 model.add(BatchNormalization())
 model.add(Dense(128, activation='relu'))
model.add(BatchNormalization())
 model.add(Dense(64, activation='relu'))
model.add(BatchNormalization())
 model.add(Dense(32, activation='relu'))
model.add(BatchNormalization())
 model.add(Dense(8, activation='relu'))
model.add(BatchNormalization())
 model.add(Dense(1))
adam = optimizers.adam(lr=LEARNING_RATE)
 model.compile(loss='mse', optimizer=adam, metrics=['mae'] )
reconstruction_loss = binary_crossentropy(inputs, outputs)
 reconstruction_loss *= original_dim
kl_loss = 1 + z_log_var - K.square(z_mean)- K.exp(z_log_var)
 kl_loss = K.sum(kl_loss, axis=-1)
 kl_loss *= -0.3
vae_loss = K.mean(reconstruction_loss + kl_loss)
 vae.add_loss(vae_loss)
 nadam = Nadam(learning_rate=0.003, beta_1=0.999, beta_2=0.999999)
 vae.compile(optimizer=nadam)
 vae.fit(x_vae_train,
epochs=epochs,
batch_size=batch_size,
shuffle=True )<predict_on_test>
print('Dataset size: %s' % DATASET_SIZE)
 print('Epochs: %s' % EPOCHS)
 print('Learning rate: %s' % LEARNING_RATE)
 print('Batch size: %s' % BATCH_SIZE)
 print('Input dimension: %s' % train_df_scaled.shape[1])
 print('Features used: %s' % train_df.columns )
decoded_train = vae.predict(x_vae_train)
 decoded_train = np.reshape(decoded_train, [-1, 32, 32, 1] )<data_type_conversions>
history = model.fit(x=train_df_scaled, y=train_labels, batch_size=BATCH_SIZE, epochs=EPOCHS,
verbose=1, validation_data=(validation_df_scaled, validation_labels),
shuffle=True )
_max = np.amax(decoded_train)
 _min = np.amin(decoded_train)
 _range = _max - _min
_step = _range / 10
decoded_train =(decoded_train * 18)
 decoded_train = decoded_train.astype(int )<predict_on_test>
prediction = model.predict(test_scaled, batch_size=128, verbose=1 )
decoded_test = vae.predict(x_vae_test)
 decoded_test = np.reshape(decoded_test, [-1, 32, 32, 1])
 _max = np.amax(decoded_test)
 _min = np.amin(decoded_test)
 _range = _max - _min
_step = _range / 10
decoded_test =(decoded_test * 19)
 decoded_test = decoded_test.astype(int)
 print(np.amax(decoded_test))
print(np.amin(decoded_test))
rd_test = []
for i in range(len(decoded_test)) :
w = ox_test[i].shape[0]
h = ox_test[i].shape[1]
if(decoded_test[i].shape[0] != h)|(decoded_test[i].shape[1] != w):
rd_test.append(cv2.resize(decoded_test[i], dsize=(h, w), interpolation=cv2.INTER_NEAREST))<load_from_csv>
output_submission(test, prediction, 'key', 'fare_amount', SUBMISSION_NAME )