Kaggle competition: New York City Taxi Fare Prediction
<SOS> metric: RMSE Kaggle data source: new-york-city-taxi-fare-prediction<import_modules>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Conv2D, MaxPooling2D, Input, Dense, LeakyReLU, Flatten, Dropout, BatchNormalization,Add,GlobalAveragePooling2D,Softmax
from tensorflow.keras.callbacks import LearningRateScheduler,ModelCheckpoint,EarlyStopping
from tensorflow.keras.preprocessing.image import ImageDataGenerator
import math
from sklearn.model_selection import train_test_split
import random<load_from_csv>
df = pd.read_csv('.. /input/train.csv',nrows=5123456, usecols=[1,2,3,4,5,6,7] )
train_datas = pd.read_csv(".. /input/Kannada-MNIST/train.csv")
 val_datas = pd.read_csv(".. /input/Kannada-MNIST/Dig-MNIST.csv" )<prepare_x_and_y>
df['pickup_datetime'] = df['pickup_datetime'].str.slice(0, 16)
 df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'], utc=False, format='%Y-%m-%d %H:%M' )
print(train_datas.shape)
 print(val_datas.shape)
 datas = pd.concat([train_datas,val_datas],axis=0)
 print(datas.shape)
 datas_X = np.array(datas.drop("label",axis=1),dtype=np.float32)
 datas_Y = np.array(datas[["label"]],dtype=np.int32)
 train_X,val_X,train_Y,val_Y = train_test_split(datas_X,datas_Y,test_size=0.1,shuffle=True )<choose_model_class>
test = pd.read_csv('.. /input/test.csv' ).set_index('key')
 test['pickup_datetime'] = test['pickup_datetime'].str.slice(0, 16)
 test['pickup_datetime'] = pd.to_datetime(test['pickup_datetime'], utc=False, format='%Y-%m-%d %H:%M')
 test.head()
tf.reset_default_graph()
nets = 1
model = [0]*nets
for j in range(nets):
input_layer = Input(shape=(28,28,1))
net = Conv2D(64,(3,3), padding='same' )(input_layer)
 net = BatchNormalization(momentum=0.5, epsilon=1e-5, gamma_initializer="uniform" )(net)
 net = LeakyReLU(alpha=0.1 )(net)
 net = Conv2D(64,(3,3), padding='same' )(net)
 net = BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer="uniform" )(net)
 net = LeakyReLU(alpha=0.1 )(net)
 net = MaxPooling2D(2, 2 )(net)
 net = Conv2D(128,(3,3), padding='same' )(net)
 net = BatchNormalization(momentum=0.2, epsilon=1e-5, gamma_initializer="uniform" )(net)
 net = LeakyReLU(alpha=0.1 )(net)
 net = Conv2D(128,(3,3), padding='same' )(net)
 net = BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer="uniform" )(net)
 net = LeakyReLU(alpha=0.1 )(net)
 net = MaxPooling2D(2,2 )(net)
 net = Dropout(0.2 )(net)
 net = Conv2D(256,(3,3), padding='same' )(net)
 net = BatchNormalization(momentum=0.2, epsilon=1e-5, gamma_initializer="uniform" )(net)
 net = LeakyReLU(alpha=0.1 )(net)
 net = Conv2D(256,(3,3), padding='same' )(net)
 net = BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer="uniform" )(net)
 net = LeakyReLU(alpha=0.1 )(net)
 net = MaxPooling2D(2,2 )(net)
 net = Dropout(0.2 )(net)
 net = GlobalAveragePooling2D()(net)
 net = BatchNormalization()(net)
 output = Dense(10, activation='softmax' )(net)
 model[j] = Model(inputs=input_layer,outputs=output)
 model[j].compile(optimizer='Adam', loss="sparse_categorical_crossentropy",metrics=["accuracy"])
 print(model[0].summary() )<choose_model_class>
def dist(pickup_lat, pickup_long, dropoff_lat, dropoff_long):
distance = np.abs(dropoff_lat - pickup_lat)+ np.abs(dropoff_long - pickup_long)
 return distance
datagen = ImageDataGenerator(
 rotation_range = 20,
width_shift_range = 0.25,
height_shift_range = 0.25,
shear_range = 0.1,
zoom_range = 0.25,
horizontal_flip = False,
rescale=1/255.0 )<choose_model_class>
def transform(data):
data['hour'] = data['pickup_datetime'].dt.hour
data['day'] = data['pickup_datetime'].dt.day
data['year'] = data['pickup_datetime'].dt.year
nyc =(-74.0063889, 40.7141667)
 jfk =(-73.7822222222, 40.6441666667)
 ewr =(-74.175, 40.69)
 lgr =(-73.87, 40.77)
 data['distance_to_center'] = dist(nyc[1], nyc[0],
data['pickup_latitude'], data['pickup_longitude'])
 data['pickup_distance_to_jfk'] = dist(jfk[1], jfk[0],
data['pickup_latitude'], data['pickup_longitude'])
 data['dropoff_distance_to_jfk'] = dist(jfk[1], jfk[0],
data['dropoff_latitude'], data['dropoff_longitude'])
 data['pickup_distance_to_lgr'] = dist(lgr[1], lgr[0],
data['pickup_latitude'], data['pickup_longitude'])
 data['dropoff_distance_to_lgr'] = dist(lgr[1], lgr[0],
data['dropoff_latitude'], data['dropoff_longitude'])
 data['long_dist'] = data['pickup_longitude'] - data['dropoff_longitude']
data['lat_dist'] = data['pickup_latitude'] - data['dropoff_latitude']
data['dist'] = dist(data['pickup_latitude'], data['pickup_longitude'],
data['dropoff_latitude'], data['dropoff_longitude'])
 return data
annealer = LearningRateScheduler(lambda x: 0.002 * 0.99 ** x)
 model_checks = []
for j in range(nets):
model_checks.append(ModelCheckpoint("model-{}.ckpt".format(j), monitor='val_acc', verbose=1, save_best_only=True,))
early_stop = EarlyStopping(monitor='val_acc', verbose=2, patience=30 )<train_model>
df = transform(df)
 test = transform(test )
history = [0] * nets
epochs = 150
batch_size = 1024
for j in range(nets):
history[j] = model[j].fit_generator(datagen.flow(train_X,train_Y, batch_size=batch_size),
epochs = epochs,
steps_per_epoch = math.ceil(train_X.shape[0]/batch_size)*2,
validation_data =(val_X/255.0,val_Y),
callbacks=[annealer,model_checks[j],early_stop],
verbose=2)
 print("Train: Model{:d}: Epochs={:d},LearningRate={:.12f} ,Train accuracy={:.5f}, Validation accuracy={:.5f}".format(
 j+1,epochs,tf.keras.backend.get_value(model[j].optimizer.lr),max(history[j].history['acc']),max(history[j].history['val_acc'])) )<load_from_csv>
df.to_csv("taxiFare5M_train.csv.gz",index=False,compression="gzip")
 test.to_csv("taxiFare5M_test.csv.gz",index=False,compression="gzip" )
test_csv = pd.read_csv(".. /input/Kannada-MNIST/test.csv")
 data_test = np.array(test_csv.drop("id",axis=1),dtype=np.float32)
 data_test = np.reshape(data_test,(-1,28,28,1))
X_test = data_test / 255.0
print(X_test.shape,X_test.dtype )<load_pretrained>
df.drop('pickup_datetime', axis=1,inplace=True)
for j in range(nets):
model[j].load_weights("model-{}.ckpt".format(j))<predict_on_test>
import xgboost as xgb
from bayes_opt import BayesianOptimization
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split
results = np.zeros(( X_test.shape[0],10))
for j in range(nets):
results = results + model[j].predict(X_test)
 results = np.argmax(results,axis = 1 )<save_to_csv>
X_train, X_test, y_train, y_test = train_test_split(df.drop('fare_amount', axis=1),
df['fare_amount'], test_size=0.25)
 del(df)
 dtrain = xgb.DMatrix(X_train, label=y_train)
 del(X_train)
 dtest = xgb.DMatrix(X_test)
 del(X_test )
submission = pd.read_csv(".. /input/Kannada-MNIST/sample_submission.csv")
 submission['label'] = results
submission.to_csv("submission.csv",index=False )<data_type_conversions>
def xgb_evaluate(max_depth, gamma, colsample_bytree):
params = {'eval_metric': 'rmse',
'max_depth': int(max_depth),
'subsample': 0.8,
'eta': 0.1,
'gamma': gamma,
'colsample_bytree': colsample_bytree}
cv_result = xgb.cv(params, dtrain, num_boost_round=60, nfold=2)
 return -1.0 * cv_result['test-rmse-mean'].iloc[-1]
def get_images_and_labels(csv,label):
fileraw = pd.read_csv(csv)
 labels = fileraw[label].to_numpy()
data = fileraw.drop([label],axis=1 ).to_numpy(dtype=np.float32)
 data = np.true_divide(data,255.).reshape(( fileraw.shape[0],28,28))
data = np.expand_dims(data, axis=1)
 return data, labels<train_model>
xgb_bo = BayesianOptimization(xgb_evaluate, {'max_depth':(3, 7),
'gamma':(0, 1),
'colsample_bytree':(0.3, 0.9)})
 xgb_bo.maximize(init_points=5, n_iter=6, acq='ei' )
plt.title(f'Training Label: {train_labels[4]}')
 plt.imshow(train_data[4,0],cmap='gray');<define_variables>
params = xgb_bo.res['max']['max_params']
params['max_depth'] = int(params['max_depth'])
 print(params )
np.random.seed(42)
 ran_10_pct_idx =(np.random.random_sample(train_labels.shape)) <.1
train_90_labels = train_labels[np.invert(ran_10_pct_idx)]
train_90_data = train_data[np.invert(ran_10_pct_idx)]
valid_10_labels = train_labels[ran_10_pct_idx]
valid_10_data = train_data[ran_10_pct_idx]<create_dataframe>
model2 = xgb.train(params, dtrain, num_boost_round=250)
 y_pred = model2.predict(dtest)
 y_train_pred = model2.predict(dtrain)
 print(np.sqrt(mean_squared_error(y_test, y_pred)))
 print(np.sqrt(mean_squared_error(y_train, y_train_pred)) )
train_ds = ArrayDataset(train_90_data,train_90_labels)
 valid_ds = ArrayDataset(valid_10_data,valid_10_labels)
 other_ds = ArrayDataset(other_data, other_labels)
 test_ds = ArrayDataset(test_data, test_labels )<train_model>
test = transform(test)
 test.drop('pickup_datetime', axis=1,inplace=True )
bs = 128
databunch = DataBunch.create(train_ds, valid_ds, test_ds=test_ds, bs=bs )<choose_model_class>
dtest = xgb.DMatrix(test)
 y_pred_test = model2.predict(dtest )
leak = 0.25
conv_drop = 0.35
lin_drop = 0.08
six_conv_architecture = nn.Sequential(
 conv_layer(1,16,stride=1,ks=3,leaky=leak),
nn.Dropout(conv_drop),
conv_layer(16,32,stride=1,ks=3,leaky=leak),
nn.Dropout(conv_drop),
AdaptiveConcatPool2d(14),
conv_layer(64,64,stride=1,ks=5,leaky=leak),
nn.Dropout(conv_drop),
AdaptiveConcatPool2d(sz=7),
conv_layer(128,128,stride=1,ks=5,leaky=leak),
nn.Dropout(conv_drop),
conv_layer(128,64,stride=1,ks=3,leaky=leak),
nn.Dropout(conv_drop),
conv_layer(64,32,stride=1,ks=3,leaky=leak),
nn.Dropout(conv_drop),
Flatten() ,
nn.Linear(1568, 50),
relu(inplace=True,leaky=leak),
nn.Dropout(lin_drop),
nn.Linear(50,25),
relu(inplace=True,leaky=leak),
nn.Dropout(lin_drop),
nn.Linear(25,10)
 )<choose_model_class>
holdout = pd.DataFrame({'key': test.index, 'fare_amount': y_pred_test})
 holdout.to_csv('submission.csv', index=False )