Kaggle competition: New York City Taxi Fare Prediction
<SOS> metric: RMSE Kaggle data source: new-york-city-taxi-fare-prediction<train_model>
%matplotlib inline
sns.set_style("white")
 warnings.filterwarnings('ignore')
history_gelu = model_gelu.fit_generator(
 train_datagen.flow(X_train, y_train, batch_size=BATCH_SIZE),
validation_data=test_datagen.flow(X_valid, y_valid, batch_size=BATCH_SIZE),
epochs=EPOCH_NUM,
callbacks=[rlr, mcp] 
)
 model_gelu.load_weights('weights.hdf5' )<train_model>
data = pd.read_csv(".. /input/new-york-city-taxi-fare-prediction/train.csv", nrows = 7000000 )
print('val_acc
relu:{}
gelu:{}
val_loss
relu:{}
gelu:{}'.format(max(history_relu.history['val_acc']), max(history_gelu.history['val_acc']), min(history_relu.history['val_loss']), min(history_gelu.history['val_loss'])) )<predict_on_test>
q1 = data['fare_amount'].quantile(0.25)
 q3 = data['fare_amount'].quantile(0.75)
 iqr = q3 - q1
print("Fare Amount lower bound : ", q1 -(1.5 * iqr),
"Fare Amount upper bound : ", q3 +(1.5 * iqr))
pred_ret = None
if min(history_relu.history['val_loss'])< min(history_gelu.history['val_loss']):
pred_ret = model_relu.predict_generator(test_datagen.flow(test_data, None, shuffle=False, batch_size=BATCH_SIZE))
else:
pred_ret = model_gelu.predict_generator(test_datagen.flow(test_data, None, shuffle=False, batch_size=BATCH_SIZE))
pred_ids = np.argmax(pred_ret, axis=1)
 pred_ids.shape<prepare_output>
print("Total null values:
", data.isnull().sum())
 print("Percentage of null values:
",
data[["dropoff_longitude", "dropoff_latitude"]].isnull().sum() / data.shape[0] )
submission['label'] = pred_ids
submission.head()<save_to_csv>
data.dropna(how='any', axis='rows', inplace=True )
submission.to_csv(path_or_buf='submission.csv', index=False )<data_type_conversions>
def get_cleaned(df):
return df[(df.fare_amount > 0)& 
(df.pickup_latitude > 35)&(df.pickup_latitude < 45)& 
(df.pickup_longitude > -80)&(df.pickup_longitude < -68)& 
(df.dropoff_latitude > 35)&(df.dropoff_latitude < 45)& 
(df.pickup_longitude > -80)&(df.dropoff_longitude < -68)& 
(df.passenger_count > 0)&(df.passenger_count < 8)]
data = get_cleaned(data)
 print(len(data))
print("Data lost after the cleaning process: ", 7000000 - len(data))
def get_data_labels(csv,label):
fileraw = pd.read_csv(csv)
 labels = fileraw[label].to_numpy()
data = fileraw.drop([label],axis=1 ).to_numpy(dtype=np.float32 ).reshape(( fileraw.shape[0],28,28))
data = np.expand_dims(data, axis=1)
 return data, labels<define_variables>
def sphere_dist(pick_lat, pick_lon, drop_lat, drop_lon):
R_earth = 6371
pick_lat, pick_lon, drop_lat, drop_lon = map(np.radians, [pick_lat, pick_lon,
drop_lat, drop_lon])
 dlat = drop_lat - pick_lat
dlon = drop_lon - pick_lon
a = np.sin(dlat/2.0)**2 + np.cos(pick_lat)* np.cos(drop_lat)* np.sin(dlon/2.0)**2
return 2 * R_earth * np.arcsin(np.sqrt(a))
train_data, train_labels = get_data_labels(DATAPATH/'train.csv','label')
 test_data, test_labels = get_data_labels(DATAPATH/'test.csv','id')
 other_data, other_labels = get_data_labels(DATAPATH/'Dig-MNIST.csv','label' )<train_model>
def datetime_info(df):
df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'],format="%Y-%m-%d %H:%M:%S UTC")
 df['hour'] = df.pickup_datetime.dt.hour
df['day'] = df.pickup_datetime.dt.day
df['month'] = df.pickup_datetime.dt.month
df['weekday'] = df.pickup_datetime.dt.weekday
df['year'] = df.pickup_datetime.dt.year
return df
data = datetime_info(data)
 data = airport_dist(data)
 data['distance'] = sphere_dist(data['pickup_latitude'], data['pickup_longitude'],
data['dropoff_latitude'], data['dropoff_longitude'])
 data.head()
plt.title(f'Training Label: {train_labels[5]}')
 plt.imshow(train_data[8,0],cmap='gray'); 
<define_variables>
data.drop(columns=["key", "pickup_datetime"], inplace=True)
 data.head()
np.random.seed(60)
 ran_10_pct_idx =(np.random.random_sample(train_labels.shape)) <.001
train_90_labels = train_labels[np.invert(ran_10_pct_idx)]
train_90_data = train_data[np.invert(ran_10_pct_idx)]
valid_10_labels = train_labels[ran_10_pct_idx]
valid_10_data = train_data[ran_10_pct_idx]<create_dataframe>
y = data["fare_amount"]
train = data.drop(columns=["fare_amount"])
 x_train, x_test, y_train, y_test = train_test_split(train, y, random_state=2666, test_size=0.05 )
train_ds = ArrayDataset(train_90_data,train_90_labels)
 valid_ds = ArrayDataset(valid_10_data,valid_10_labels)
 other_ds = ArrayDataset(other_data, other_labels)
 test_ds = ArrayDataset(test_data, test_labels )<train_model>
params = {
"max_depth": 7,
"subsample": 0.9,
"eta": 0.03,
"colsample_bytree": 0.9,
"random_state": 2666,
"objective": "reg:linear",
"eval_metric": "rmse",
"silent": 1
}
bs = 256
databunch = DataBunch.create(train_ds, valid_ds, test_ds=test_ds, bs=bs )<concatenate>
def XGBmodel(x_train, x_test, y_train, y_test, params):
matrix_train = xgb.DMatrix(x_train, label=y_train)
 matrix_test = xgb.DMatrix(x_test, label=y_test)
 model = xgb.train(params=params,
dtrain=matrix_train,num_boost_round=5000,
early_stopping_rounds=10,evals=[(matrix_test,'test')])
 return model
start_time = time.time()
model = XGBmodel(x_train, x_test, y_train, y_test, params )
def conv2(ni,nf,stride=2,ks=5): return conv_layer(ni,nf,stride=stride,ks=ks)
<choose_model_class>
time_taken = time.time() - start_time
time_taken
best_architecture = nn.Sequential(
 conv2(1,32,stride=1,ks=5),
conv2(32,32,stride=1,ks=5),
conv2(32,32,stride=2,ks=8),
nn.Dropout(0.2),
conv2(32,64,stride=1,ks=5),
conv2(64,64,stride=1,ks=5),
conv2(64,64,stride=2,ks=5),
nn.Dropout(0.6),
Flatten() ,
nn.Linear(3136, 256),
relu(inplace=True),
nn.BatchNorm1d(256),
nn.Dropout(0.2),
nn.Linear(256,10)
 )<choose_model_class>
test = pd.read_csv('.. /input/new-york-city-taxi-fare-prediction/test.csv')
 test = datetime_info(test)
 test = airport_dist(test)
 test['distance'] = sphere_dist(test['pickup_latitude'], test['pickup_longitude'],
test['dropoff_latitude'] , test['dropoff_longitude'])
 test_key = test['key']
x_pred = test.drop(columns=['key', 'pickup_datetime'])
 prediction = model.predict(xgb.DMatrix(x_pred), ntree_limit=model.best_ntree_limit )
<train_model><EOS> 
submission = pd.DataFrame({
"key": test_key,
"fare_amount": prediction.round(2)
 })
 submission.to_csv('taxi_fare_submission.csv',index=False)
 submission.head()