Kaggle competition: New York City Taxi Fare Prediction
<SOS> metric: RMSE Kaggle data source: new-york-city-taxi-fare-prediction<feature_engineering>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import geopy.distance as geo
import datetime
import time
import calendar
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report,confusion_matrix
from sklearn import preprocessing
from sklearn import metrics
train_df['date'] = train_df['timestamp'].dt.date
train_df['meter_reading_log1p'] = np.log1p(train_df['meter_reading'] )<count_values>
%matplotlib inline
np.sum(train_df['meter_reading_log1p'].values < 0 )<filter>
df = pd.read_csv('.. /input/train.csv', nrows = 2000000 )
building_meta_df[building_meta_df.site_id == 0]<filter>
unwanted_indices = df[(abs(df['pickup_latitude'])> 90)| 
(abs(df['dropoff_latitude'])> 90)
 ].index
df.drop(list(unwanted_indices), inplace=True)
 del unwanted_indices
unwanted_indices = df[(abs(df['pickup_longitude'])> 180)| 
(abs(df['dropoff_longitude'])> 180)
 ].index
df.drop(list(unwanted_indices), inplace=True )
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")' )<define_variables>
df[(abs(df['pickup_longitude'])> 180)| 
(abs(df['dropoff_longitude'])> 180)
 ].index
debug = False<feature_engineering>
df[(abs(df['pickup_latitude'])> 90)| 
(abs(df['dropoff_latitude'])> 90)
 ].index
def preprocess(df):
df["hour"] = df["timestamp"].dt.hour
df["month"] = df["timestamp"].dt.month
df["dayofweek"] = df["timestamp"].dt.dayofweek
df["weekend"] = df["dayofweek"] >= 5 
<categorify>
unwanted_indices = df[df['dropoff_latitude'].isnull() ].index
df.drop(list(unwanted_indices), inplace=True)
 unwanted_indices = df[df['dropoff_longitude'].isnull() ].index
df.drop(list(unwanted_indices), inplace=True )
preprocess(train_df )<categorify>
if 'unwanted_indices' in globals() : del unwanted_indices
unwanted_indices = df[(df['passenger_count'] > 6)|(df['passenger_count'] == 0)].index
df.drop(list(unwanted_indices), inplace=True )
<data_type_conversions>
unwanted_indices = df[df['fare_amount']<=0].index
df.drop(list(unwanted_indices), inplace=True )
df_group = train_df.groupby(['building_id', 'meter'])['meter_reading_log1p']
building_mean = df_group.mean().astype(np.float16)
 building_median = df_group.median().astype(np.float16)
 building_min = df_group.min().astype(np.float16)
 building_max = df_group.max().astype(np.float16)
 building_std = df_group.std().astype(np.float16 )<merge>
building_stats_df = pd.concat([building_mean, building_median, building_min, building_max, building_std], axis=1,
keys=['building_mean', 'building_median', 'building_min', 'building_max', 'building_std'] ).reset_index()
train_df = pd.merge(train_df, building_stats_df, on=['building_id', 'meter'], how='left', copy=False )<count_missing_values>
df['Travel distance'] = list(map(lambda x1,x2,x3,x4:
geo.distance(( x3,x1),(x4,x2)).miles,
df['pickup_longitude'], df['dropoff_longitude'],
df['pickup_latitude'], df['dropoff_latitude']))
weather_train_df.isna().sum()<load_from_disk>
newtimeStamp = pd.to_datetime(df['pickup_datetime'].apply(lambda x: x.split(' UTC')[0]))
weather_test_df = pd.read_feather(root/'weather_test.feather')
 weather = pd.concat([weather_train_df, weather_test_df],ignore_index=True)
 del weather_test_df
weather_key = ['site_id', 'timestamp']<feature_engineering>
df['Hour'] = newtimeStamp.apply(lambda x : x.hour)
 df['Month'] = newtimeStamp.apply(lambda x : x.month)
 df['Date'] = newtimeStamp.apply(lambda x : x.date() )
temp_skeleton = weather[weather_key + ['air_temperature']].drop_duplicates(subset=weather_key ).sort_values(by=weather_key ).copy()
temp_skeleton['temp_rank'] = temp_skeleton.groupby(['site_id', temp_skeleton.timestamp.dt.date])['air_temperature'].rank('average')
 df_2d = temp_skeleton.groupby(['site_id', temp_skeleton.timestamp.dt.hour])['temp_rank'].mean().unstack(level=1)
 site_ids_offsets = pd.Series(df_2d.values.argmax(axis=1)- 14)
 site_ids_offsets.index.name = 'site_id'
def timestamp_align(df):
df['offset'] = df.site_id.map(site_ids_offsets)
 df['timestamp_aligned'] =(df.timestamp - pd.to_timedelta(df.offset, unit='H'))
df['timestamp'] = df['timestamp_aligned']
del df['timestamp_aligned']
return df
del weather
del temp_skeleton
gc.collect()<concatenate>
for x in df['pickup_datetime'] :
y = x.split(' ')[2]
if(y!='UTC'):
print(y )
weather_train_df = timestamp_align(weather_train_df)
 weather_train_df = weather_train_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))<data_type_conversions>
Day_of_Week=newtimeStamp.apply(
 lambda x :
calendar.day_name[datetime.date(x.year,x.month,x.day)
.weekday() ] 
)
 dmap = {0:'Mon',1:'Tue',2:'Wed',3:'Thu',4:'Fri',5:'Sat',6:'Sun'}
Day_of_Week = newtimeStamp.apply(
 lambda x :
dmap[datetime.date(x.year,x.month,x.day)
.weekday() ] 
)
 df['Day of Week'] = Day_of_Week
def add_lag_feature(weather_df, window=3):
group_df = weather_df.groupby('site_id')
 cols = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction', 'wind_speed']
rolled = group_df[cols].rolling(window=window, min_periods=0)
 lag_mean = rolled.mean().reset_index().astype(np.float16)
 lag_max = rolled.max().reset_index().astype(np.float16)
 lag_min = rolled.min().reset_index().astype(np.float16)
 lag_std = rolled.std().reset_index().astype(np.float16)
 for col in cols:
weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]
weather_df[f'{col}_max_lag{window}'] = lag_max[col]
weather_df[f'{col}_min_lag{window}'] = lag_min[col]
weather_df[f'{col}_std_lag{window}'] = lag_std[col]<feature_engineering>
df['qty'] = df['pickup_datetime'].apply(lambda x: 1 )
add_lag_feature(weather_train_df, window=3)
 add_lag_feature(weather_train_df, window=72 )<categorify>
df['timestamp'] = newtimeStamp.apply(lambda x : time.mktime(
(x.year,x.month,x.day,
x.hour,x.minute,x.second,
1,x.day,-1))
)
primary_use_list = building_meta_df['primary_use'].unique()
primary_use_dict = {key: value for value, key in enumerate(primary_use_list)}
print('primary_use_dict: ', primary_use_dict)
 building_meta_df['primary_use'] = building_meta_df['primary_use'].map(primary_use_dict)
 gc.collect()<define_variables>
dfm=df.groupby(['Day of Week','Hour'] ).count()
fp=dfm['qty'].unstack()
category_cols = ['building_id', 'site_id', 'primary_use']
weather_cols = [
'air_temperature', 'cloud_coverage',
'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
'wind_direction', 'wind_speed', 'air_temperature_mean_lag72',
'air_temperature_max_lag72', 'air_temperature_min_lag72',
'air_temperature_std_lag72', 'cloud_coverage_mean_lag72',
'dew_temperature_mean_lag72', 'precip_depth_1_hr_mean_lag72',
'sea_level_pressure_mean_lag72', 'wind_direction_mean_lag72',
'wind_speed_mean_lag72', 'air_temperature_mean_lag3',
'air_temperature_max_lag3',
'air_temperature_min_lag3', 'cloud_coverage_mean_lag3',
'dew_temperature_mean_lag3',
'precip_depth_1_hr_mean_lag3', 'sea_level_pressure_mean_lag3',
'wind_direction_mean_lag3', 'wind_speed_mean_lag3']
feature_cols = ['square_feet', 'year_built'] + [
'hour', 'weekend', 'dayofweek',
'building_median'] + weather_cols<prepare_x_and_y>
dftrain = df.drop([
'key',
'pickup_datetime',
'Hour',
'Month',
'Date',
'Day of Week',
'qty'
],axis=1 ).dropna()
dftrain.head(2 )
def create_X_y(train_df, target_meter):
target_train_df = train_df[train_df['meter'] == target_meter]
target_train_df = target_train_df.merge(building_meta_df, on='building_id', how='left')
 target_train_df = target_train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 X_train = target_train_df[feature_cols + category_cols]
y_train = target_train_df['meter_reading_log1p'].values
del target_train_df
return X_train, y_train<import_modules>
X_train = dftrain.drop('fare_amount',axis=1)
 y_train = dftrain['fare_amount']
optuna.__version__<split>
XX_train, XX_test, yy_train, yy_test = train_test_split(X_train, y_train, test_size=0.30 )
def objective(trial: Trial, fast_check=True, target_meter=0, return_info=False):
folds = 5
seed = 666
shuffle = False
kf = KFold(n_splits=folds, shuffle=shuffle, random_state=seed)
 X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models = []
valid_score = 0
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(trial, train_data, valid_data, cat_features=category_cols,
num_rounds=1000)
 y_valid_pred_total[valid_idx] = y_pred_valid
models.append(model)
 gc.collect()
valid_score += log["valid/l2"]
if fast_check:
break
valid_score /= len(models)
 if return_info:
return valid_score, models, y_pred_valid, y_train
else:
return valid_score<train_model>
def fit_lgbm(trial, train, val, devices=(-1,), seed=None, cat_features=None, num_rounds=1500):

 X_train, y_train = train
X_valid, y_valid = val
metric = 'l2'
params = {
'num_leaves': trial.suggest_int('num_leaves', 2, 256),
'objective': 'regression',
'learning_rate': 0.1,
"boosting": "gbdt",
'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),
'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),
"bagging_freq": 5,
"bagging_fraction": trial.suggest_uniform('bagging_fraction', 0.1, 1.0),
"feature_fraction": trial.suggest_uniform('feature_fraction', 0.4, 1.0),
"metric": metric,
"verbosity": -1,
}
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 params['seed'] = seed
early_stop = 20
verbose_eval = 20
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 print('best_score', model.best_score)
 log = {'train/l2': model.best_score['training']['l2'],
'valid/l2': model.best_score['valid_1']['l2']}
return model, y_pred_valid, log<find_best_params>
study = optuna.create_study()
study.optimize(objective, n_trials=10 )<train_model>
def fit_lgbm_with_pruning(trial, train, val, devices=(-1,), seed=None, cat_features=None, num_rounds=1500):

 X_train, y_train = train
X_valid, y_valid = val
metric = 'l2'
params = {
'num_leaves': trial.suggest_int('num_leaves', 2, 256),
'objective': 'regression',
'learning_rate': 0.1,
"boosting": "gbdt",
'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),
'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),
"bagging_freq": 5,
"bagging_fraction": trial.suggest_uniform('bagging_fraction', 0.1, 1.0),
"feature_fraction": trial.suggest_uniform('feature_fraction', 0.4, 1.0),
"metric": metric,
"verbosity": -1,
}
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 params['seed'] = seed
early_stop = 20
verbose_eval = 20
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
pruning_callback = optuna.integration.LightGBMPruningCallback(trial, 'l2', valid_name='valid_1')
 print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop,
callbacks=[pruning_callback])
 y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 print('best_score', model.best_score)
 log = {'train/l2': model.best_score['training']['l2'],
'valid/l2': model.best_score['valid_1']['l2']}
return model, y_pred_valid, log<compute_train_metric>
def objective_with_prune(trial: Trial, fast_check=True, target_meter=0):

 folds = 5
seed = 666
shuffle = False
kf = KFold(n_splits=folds, shuffle=shuffle, random_state=seed)
 X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models0 = []
valid_score = 0
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm_with_pruning(trial, train_data, valid_data, cat_features=category_cols,
num_rounds=1000)
 y_valid_pred_total[valid_idx] = y_pred_valid
models0.append(model)
 gc.collect()
valid_score += log["valid/l2"]
if fast_check:
break
valid_score /= len(models0)
 return valid_score<find_best_params>
dftest = pd.read_csv('.. /input/test.csv' )
study = optuna.create_study(pruner=optuna.pruners.MedianPruner(n_warmup_steps=5))
study.optimize(objective_with_prune, n_trials=50 )<train_model>
newtimeStamp=pd.to_datetime(dftest['pickup_datetime'])
 dftest['Hour'] = newtimeStamp.apply(lambda x : x.hour)
 dftest['Month'] = newtimeStamp.apply(lambda x : x.month)
 dftest['Date'] = newtimeStamp.apply(lambda x : x.date() )
print('Best trial: score {}, params {}'.format(study.best_trial.value, study.best_trial.params)) 
<create_dataframe>
Day_of_Week=newtimeStamp.apply(
 lambda x :
calendar.day_name[datetime.date(x.year,x.month,x.day)
.weekday() ] 
)
 dmap = {0:'Mon',1:'Tue',2:'Wed',3:'Thu',4:'Fri',5:'Sat',6:'Sun'}
Day_of_Week = newtimeStamp.apply(
 lambda x :
dmap[datetime.date(x.year,x.month,x.day)
.weekday() ] 
)
 dftest['Day of Week'] = Day_of_Week
trials_df = study.trials_dataframe()
trials_df<find_best_params>
dftest['qty'] = dftest['pickup_datetime'].apply(lambda x: 1 )
study = optuna.create_study(pruner=optuna.pruners.SuccessiveHalvingPruner(min_resource=2, reduction_factor=4, min_early_stopping_rate=1))
study.optimize(objective_with_prune, timeout=timeout )<find_best_params>
dftest['timestamp'] = newtimeStamp.apply(lambda x : time.mktime(
(x.year,x.month,x.day,
x.hour,x.minute,x.second,
1,x.day,-1)) )
print(f'Executed {len(study.trials)} trials, best score {study.best_value} with best_params {study.best_params}' )<save_to_csv>
dftest['Travel distance'] = list(map(lambda x1,x2,x3,x4:
geo.distance(( x3,x1),(x4,x2)).miles,
dftest['pickup_longitude'], dftest['dropoff_longitude'],
dftest['pickup_latitude'], dftest['dropoff_latitude']))
trials_df = study.trials_dataframe()
trials_df.to_csv('trials_history.csv' )<train_model>
dfm=dftest.groupby(['Day of Week','Hour'] ).count()
fp=dfm['qty'].unstack()
fp
print("Training model with best_params {}".format(study.best_params))<drop_column>
from sklearn import preprocessing
try:
del train_df
del train_df_original
del weather_train_df
except:
pass
gc.collect()<feature_engineering>
coltest=list(dftest.columns)
 idy=coltest[2:7]
idy.append(coltest[12])
 idy.append(coltest[13])
 idy
print('loading...')
 test_df = pd.read_feather(root/'test.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 print('preprocessing building...')
 test_df['date'] = test_df['timestamp'].dt.date
preprocess(test_df)
 print('preprocessing weather...')
 weather_test_df = timestamp_align(weather_test_df)
 weather_test_df = weather_test_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
weather_test_df.groupby('site_id' ).apply(lambda group: group.isna().sum())
 add_lag_feature(weather_test_df, window=3)
 add_lag_feature(weather_test_df, window=72)
 print('reduce mem usage...')
 weather_test_df = weather_test_df[['site_id', 'timestamp'] + weather_cols]
reduce_mem_usage(test_df, use_float16=True)
 reduce_mem_usage(weather_test_df, use_float16=True)
 gc.collect()<drop_column>
X_train = dftrain[idy]
y_train = dftrain['fare_amount']
X_test = dftest[idy]
del df_2d
del site_ids_offsets
gc.collect()<merge>
def create_X(test_df, target_meter):
target_test_df = test_df[test_df['meter'] == target_meter]
target_test_df = target_test_df.merge(building_meta_df, on='building_id', how='left', copy=False)
 target_test_df = pd.merge(target_test_df, building_stats_df, on=['building_id', 'meter'], how='left', copy=False)
 target_test_df = target_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left', copy=False)
 X_test = target_test_df[feature_cols + category_cols]
return X_test
def pred(X_test, models, batch_size=1000000):
iterations =(X_test.shape[0] + batch_size -1)// batch_size
print('iterations', iterations)
 y_test_pred_total = np.zeros(X_test.shape[0])
 for i, model in enumerate(models):
print(f'predicting {i}-th model')
 for k in tqdm(range(iterations)) :
y_pred_test = model.predict(X_test[k*batch_size:(k+1)*batch_size], num_iteration=model.best_iteration)
 y_test_pred_total[k*batch_size:(k+1)*batch_size] += y_pred_test
y_test_pred_total /= len(models)
 return y_test_pred_total<predict_on_test>
%%time
X_test = create_X(test_df, target_meter=0)
 gc.collect()
y_test0 = pred(X_test, models0)
 del X_test
gc.collect()<compute_test_metric>
scaler = sklearn.preprocessing.StandardScaler()
gbm = lgb.LGBMRegressor(objective='regression',
boosting_type='gbdt',
num_leaves=1001,
learning_rate=0.01,
n_estimators=2500)
 steps = [('scaler', scaler), 
('GBM', gbm)]
pipeline = sklearn.pipeline.Pipeline(steps)
 pipeline.fit(X_train, y_train)
 predictions = pipeline.predict(X_test )
%%time
X_test = create_X(test_df, target_meter=1)
 gc.collect()
y_test1 = pred(X_test, models1)
 del X_test
gc.collect()<compute_test_metric>
fare = []
for y_pred in predictions:
fare.append('{:.{prec}f}'.format(y_pred, prec=2))
%%time
X_test = create_X(test_df, target_meter=2)
 gc.collect()
y_test2 = pred(X_test, models2)
 del X_test
gc.collect()<compute_test_metric>
pd.DataFrame({ 'key':list(dftest['key']),
'fare_amount':fare } ).set_index('key' ).to_csv('sample_submission.csv', sep=',' )