Kaggle competition: New York City Taxi Fare Prediction
<SOS> metric: RMSE Kaggle data source: new-york-city-taxi-fare-prediction<merge>
%matplotlib inline
seed = 42
building_meta_df = building_meta_df.merge(train_df[['building_id','meter']].drop_duplicates() , on='building_id' )<data_type_conversions>
train = pd.read_csv('.. /input/train.csv', nrows = 8000000, parse_dates = ['pickup_datetime'] )
building_meta_df['groupNum_train'] = building_meta_df['site_id'].astype('int')*10 + building_meta_df['meter'].astype('int' )<filter>
train = train.dropna()
train_df = train_df [ train_df['building_id'] != 1099 ]<feature_engineering>
train = train[select_within_boundingbox(train, BB)]
building_meta_df['floor_area'] = building_meta_df.square_feet / building_meta_df.floor_count<feature_engineering>
train = train[train['fare_amount'].between(left = 2.50, right = 100)]
train = train.loc[train['passenger_count'] < 10]
en_holidays = holidays.England()
ir_holidays = holidays.Ireland()
ca_holidays = holidays.Canada()
us_holidays = holidays.UnitedStates()
def add_holiyday(df_weather):
en_idx = df_weather.query('site_id == 1 or site_id == 5' ).index
ir_idx = df_weather.query('site_id == 12' ).index
ca_idx = df_weather.query('site_id == 7 or site_id == 11' ).index
us_idx = df_weather.query('site_id == 0 or site_id == 2 or site_id == 3 or site_id == 4 or site_id == 6 or site_id == 8 or site_id == 9 or site_id == 10 or site_id == 13 or site_id == 14 or site_id == 15' ).index
df_weather['IsHoliday'] = 0
df_weather.loc[en_idx, 'IsHoliday'] = df_weather.loc[en_idx, 'timestamp'].apply(lambda x: en_holidays.get(x, default=0))
df_weather.loc[ir_idx, 'IsHoliday'] = df_weather.loc[ir_idx, 'timestamp'].apply(lambda x: ir_holidays.get(x, default=0))
df_weather.loc[ca_idx, 'IsHoliday'] = df_weather.loc[ca_idx, 'timestamp'].apply(lambda x: ca_holidays.get(x, default=0))
df_weather.loc[us_idx, 'IsHoliday'] = df_weather.loc[us_idx, 'timestamp'].apply(lambda x: us_holidays.get(x, default=0))
holiday_idx = df_weather['IsHoliday'] != 0
df_weather.loc[holiday_idx, 'IsHoliday'] = 1
df_weather['IsHoliday'] = df_weather['IsHoliday'].astype(np.uint8 )<feature_engineering>
def transform_time(df):
df['year'] = df['pickup_datetime'].dt.year
df['month'] = df['pickup_datetime'].dt.month
df['day'] = df['pickup_datetime'].dt.day
df['hour'] = df['pickup_datetime'].dt.hour
df['minute'] = df['pickup_datetime'].dt.minute
df['second'] = df['pickup_datetime'].dt.second
df['weekday'] = df['pickup_datetime'].dt.weekday
df['PM'] =(df['hour'] >= 12 ).apply(int)
 df['dayofyear'] = df['pickup_datetime'].dt.dayofyear
df['daysinmonth'] = df['pickup_datetime'].dt.daysinmonth
df['is_leap_year'] =(df['pickup_datetime'].dt.is_leap_year ).apply(int)
 df['weekofyear'] = df['pickup_datetime'].dt.weekofyear
df['dayofweek'] = df['pickup_datetime'].dt.dayofweek
df['daysinyear'] = df['is_leap_year'].apply(lambda x : 366 if x == 1 else 365)
 df['frac_day'] =(df['hour'] +(df['minute'] / 60)+(df['second'] / 3600)) / 24
df['frac_week'] =(df['dayofweek'] + df['frac_day'])/ 7
df['frac_month'] =(df['day'] +(df['frac_day'])) /(df['daysinmonth'] + 1)
 df['frac_year'] =(df['dayofyear'] + df['frac_day'])/(df['daysinyear'] + 1)
 return df
set_local(weather_train_df)
 add_holiyday(weather_train_df )<merge>
def chebyshev(pickup_long, dropoff_long, pickup_lat, dropoff_lat):
return np.maximum(np.absolute(pickup_long - dropoff_long), np.absolute(pickup_lat - dropoff_lat))
def haversine(pickup_long, dropoff_long, pickup_lat, dropoff_lat):
dlon =(pickup_long - dropoff_long)* 0.0174533
dlat =(pickup_lat - dropoff_lat)* 0.0174533
lat1 = dropoff_lat
lat2 = pickup_lat
return 2.0 * 6371.0 * np.arcsin(np.power(np.power(np.sin(dlat/2.0),2)+ np.cos(lat1)* np.cos(lat2)* np.power(np.sin(dlon/2.0),2), 0.5))
def rotate_long(long, lat):
degree = 36.1
radian = degree * 0.0174533
return long * np.cos(radian)- lat * np.sin(radian)
 def rotate_lat(long, lat):
degree = 36.1
radian = degree * 0.0174533
return long * np.sin(radian)- lat * np.cos(radian )
train_df_black = train_df.copy()
train_df_black = train_df_black.merge(building_meta_df, on=['building_id', 'meter'], how='left')
 train_df_black = train_df_black.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 train_df_black['black_count']=0
for bid in train_df_black.building_id.unique() :
df = train_df_black[train_df_black.building_id==bid]
for meter in df.meter.unique() :
dfm = df[df.meter == meter]
b =(dfm.meter_reading == 0 ).astype(int)
 train_df_black.loc[(train_df_black.building_id==bid)&(train_df_black.meter == meter), 'black_count'] = b.groupby(( ~b.astype(bool)).cumsum() ).cumsum()<merge>
def transform_distance(df):
df['abs_lat_diff'] =(df['dropoff_latitude'] - df['pickup_latitude'] ).abs()
df['abs_lon_diff'] =(df['dropoff_longitude'] - df['pickup_longitude'] ).abs()
df['Chebyshev'] = chebyshev(df['pickup_longitude'], df['dropoff_longitude'], df['pickup_latitude'], df['dropoff_latitude'])
 df['Haversine'] = haversine(df['pickup_longitude'], df['dropoff_longitude'], df['pickup_latitude'], df['dropoff_latitude'])
 nyc =(-74.0063889, 40.7141667)
 jfk =(-73.7822222222, 40.6441666667)
 ewr =(-74.175, 40.69)
 lgr =(-73.87, 40.77)
 df['cheb_nyc'] = chebyshev(nyc[0], df['dropoff_longitude'], nyc[1], df['dropoff_latitude'])
 df['cheb_jfk'] = chebyshev(jfk[0], df['dropoff_longitude'], jfk[1], df['dropoff_latitude'])
 df['cheb_ewr'] = chebyshev(ewr[0], df['dropoff_longitude'], ewr[1], df['dropoff_latitude'])
 df['cheb_lgr'] = chebyshev(lgr[0], df['dropoff_longitude'], lgr[1], df['dropoff_latitude'])
 df['hav_nyc'] = haversine(nyc[0], df['dropoff_longitude'], nyc[1], df['dropoff_latitude'])
 df['hav_jfk'] = haversine(jfk[0], df['dropoff_longitude'], jfk[1], df['dropoff_latitude'])
 df['hav_ewr'] = haversine(ewr[0], df['dropoff_longitude'], ewr[1], df['dropoff_latitude'])
 df['hav_lgr'] = haversine(lgr[0], df['dropoff_longitude'], lgr[1], df['dropoff_latitude'])
 df['rotate_pickup_longitude'] = rotate_long(df['pickup_longitude'], df['pickup_latitude'])
 df['rotate_pickup_latitude'] = rotate_lat(df['pickup_longitude'], df['pickup_latitude'])
 df['rotate_dropoff_longitude'] = rotate_long(df['dropoff_longitude'], df['dropoff_latitude'])
 df['rotate_dropoff_latitude'] = rotate_lat(df['dropoff_longitude'], df['dropoff_latitude'])
 return df
train_df = train_df.merge(train_df_black[['timestamp','building_id','meter','black_count']], on=['timestamp','building_id','meter'])
 train_df = train_df[train_df['black_count'] < 24*black_day].drop('black_count', axis=1 )<set_options>
train = transform_time(train)
 train = transform_distance(train)
 train.drop(['pickup_datetime', 'key'], axis = 1, inplace = True)
 target = train.pop('fare_amount' )
del train_df_black
gc.collect()<filter>
X_train, X_val, y_train, y_val = train_test_split(train, target, test_size = 0.2, random_state = seed )
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")' )<data_type_conversions>
reg = xgb.XGBRegressor(eval_metric = 'rmse',
max_depth = 7,
subsample = 0.8,
learning_rate = 0.1,
gamma = 1,
colsample_bytree = 0.9,
random_state = seed,
n_jobs = -1 )
( train_df[train_df.building_id == 954].meter_reading==0 ).astype(int ).plot()<filter>
reg.fit(X_train, y_train )
train_df = train_df.query('not(building_id == 954 & meter_reading == 0)')
 train_df = train_df.query('not(building_id == 1221 & meter_reading == 0)' )<feature_engineering>
train_pred = reg.predict(X_train)
 val_pred = reg.predict(X_val)
 rmse_train = np.sqrt(mean_squared_error(y_train, train_pred))
print('Training RMSE:', rmse_train)
 rmse_val = np.sqrt(mean_squared_error(y_val, val_pred))
print('Validation RMSE:', rmse_val )
funny_bids = [993, 1168, 904, 954, 778, 1021]
print('before', len(train_df))
if outlier:
train_df.loc[(train_df.building_id == 993)&(train_df.meter == 0)&(train_df.meter_reading > 30000), 'meter_reading'] = 31921
train_df.loc[(train_df.building_id == 993)&(train_df.meter == 1)&(train_df.meter_reading > 90000), 'meter_reading'] = 96545.5
train_df = train_df[(( train_df.building_id == 1168)&(train_df.meter == 0)&(train_df.meter_reading >10000)) == False]
train_df.loc[(train_df.building_id == 904)&(train_df.meter == 0)&(train_df.meter_reading >10000), 'meter_reading'] = 11306
train_df = train_df[(( train_df.building_id == 954)&(train_df.meter_reading >10000)) ==False]
if rescale:
train_df.loc[(train_df.building_id == 778)&(train_df.meter == 1), 'meter_reading'] = train_df.loc[(train_df.building_id == 778)&(train_df.meter == 1), 'meter_reading']/ 100
train_df.loc[(train_df.building_id == 1021)&(train_df.meter == 3), 'meter_reading'] = train_df.loc[(train_df.building_id == 1021)&(train_df.meter == 3), 'meter_reading']/ 1000
train_df = train_df.reset_index()
print('after', len(train_df))
gc.collect()<feature_engineering>
test = pd.read_csv('.. /input/test.csv', parse_dates = ['pickup_datetime'])
 test = transform_time(test)
 test = transform_distance(test)
 test.drop(['pickup_datetime'], axis = 1, inplace = True)
 test_index = test.pop('key' )
train_df.loc[(train_df.building_id.isin(site_0_bids)) &(train_df.meter==0), 'meter_reading'] = train_df[(train_df.building_id.isin(site_0_bids)) &(train_df.meter==0)]['meter_reading'] * 0.2931<feature_engineering>
result = reg.predict(test )
<feature_engineering><EOS> 
submission = pd.DataFrame({
'key' : test_index,
'fare_amount' : result.flatten()
})
 submission.to_csv('solution.csv', index = False )