Kaggle competition: New York City Taxi Fare Prediction
<SOS> metric: RMSE Kaggle data source: new-york-city-taxi-fare-prediction<define_search_space>
import numpy as np
import pandas as pd
import os
import math
from sklearn.model_selection import train_test_split
from sklearn import linear_model
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error
from sklearn.ensemble import GradientBoostingRegressor
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
BATCHSIZE = 50000
SEED = 529
SELECT = True
SPLITS = 5
fe_config =[ 
(True, [5000, 10000, 50000], True, True, False),
]<define_variables>
train_path = '.. /input/new-york-city-taxi-fare-prediction/train.csv'
test_path = '.. /input/new-york-city-taxi-fare-prediction/test.csv'
def seed_everything(seed : int)-> NoReturn :
rn.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 seed_everything(SEED)
<load_from_csv>
train_data = pd.read_csv(train_path, nrows=5000000)
 train_data.head()
def read_data() -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
train = pd.read_csv('.. /input/data-without-drift/train_clean.csv', dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int8})
 test = pd.read_csv('.. /input/data-without-drift/test_clean.csv', dtype={'time': np.float32, 'signal': np.float32})
 sub = pd.read_csv('.. /input/liverpool-ion-switching/sample_submission.csv', dtype={'time': np.float32})
 return train, test, sub 
<define_variables>
train_data['pickup_datetime'] = pd.to_datetime(train_data['pickup_datetime'],format='%Y-%m-%d %H:%M:%S %Z')
 train_data.head()
tr_visual_idx = [0, 500000, 600000, 1000000, 1500000, 2000000, 2500000, 3000000, 3500000, 4000000, 4500000, 5000000]
te_visual_idx = [0, 100000, 200000, 300000, 400000, 500000, 600000, 700000, 800000, 900000, 1000000, 1500000, 2000000]
tr_data_type = [0,0,0,1,2,4,3,1,2,3,4]
te_data_type = [0,2,3,0,1,4,3,4,0,2,0,0]
def add_visual_batching(df: pd.DataFrame, train: bool = True):
idx = tr_visual_idx if train else te_visual_idx
type_ = tr_data_type if train else te_data_type
s = idx; t = type_
visual_batch = np.zeros(( s[-1],), dtype=np.int64)
 visual_type = np.zeros(( s[-1],), dtype=np.int64)
 df['visual_batch'] = 0
df['visual_type'] = 0
for i, j in zip(range(len(s)- 1), t):
df.loc[s[i]:s[i+1],'visual_batch'] = i
df.loc[s[i]:s[i+1],'visual_type'] = j
return df<categorify>
train_data['pickup_year'] = train_data['pickup_datetime'].dt.year
train_data['pickup_quarter'] = train_data['pickup_datetime'].dt.quarter
train_data['pickup_month'] = train_data['pickup_datetime'].dt.month
train_data['pickup_day'] = train_data['pickup_datetime'].dt.day
train_data['pickup_hour'] = train_data['pickup_datetime'].dt.hour
train_data.head()
def lag_with_pct_change(df : pd.DataFrame,
shift_sizes : Optional[List]=[1, 2, 3],
add_pct_change : Optional[bool]=False,
add_pct_change_lag : Optional[bool]=False)-> pd.DataFrame:
df['batch'] = df.index // 500000
df['batch_idx'] = df.index -(df.batch * 500000)
 smooth_sig = lambda x: filter_wave(x, cutoff=(0, 100), filtering='filtfilt')
 df['baseline'] = np.concatenate(df.groupby(['visual_batch'])['signal'].apply(smooth_sig))
for shift_size in shift_sizes:
df['signal_shift_pos_'+str(shift_size)] = df.groupby(['batch'] ).shift(shift_size, fill_value=0.) ['signal']
df['signal_shift_neg_'+str(shift_size)] = df.groupby(['batch'] ).shift(-1*shift_size, fill_value=0.) ['signal']
if add_pct_change:
df['pct_change'] = df['signal'].pct_change()
if add_pct_change_lag:
for shift_size in shift_sizes:
df['pct_change_shift_pos_'+str(shift_size)] = df.groupby(['batch'] ).shift(shift_size, fill_value=0.) ['pct_change']
df['pct_change_shift_neg_'+str(shift_size)] = df.groupby(['batch'] ).shift(-1*shift_size, fill_value=0.) ['pct_change']
return df 
<prepare_x_and_y>
train_data.drop(['key', 'pickup_datetime'], axis=1, inplace=True)
 train_data.head()
def feature_enginering_by_batch(z : Union[pd.Series, np.array],
window_size : Optional[List]=[50000])-> pd.DataFrame:
temp = pd.DataFrame(index=[0], dtype=np.float16)
 if window_size is not None:
for window in window_size:
roll = pd.Series(z ).rolling(window=window, min_periods=1, center=True)
 temp[f'roll_mean_{window}'] = roll.mean()
temp[f'roll_max_{window}'] = roll.max()
temp[f'roll_min_{window}'] = roll.min()
temp[f'roll_std_{window}'] = roll.std()
temp[f'roll_mean_abs_chg_{window}'] = roll.apply(lambda x: np.mean(np.abs(np.diff(x))), raw=True)
 temp[f'roll_abs_max_{window}'] = roll.apply(lambda x: np.max(np.abs(x)) , raw=True)
 temp[f'roll_abs_min_{window}'] = roll.apply(lambda x: np.min(np.abs(x)) , raw=True)
 temp[f'roll_range_{window}'] = temp[f'roll_max_{window}'] - temp[f'roll_min_{window}']
temp[f'roll_max_to_min_{window}'] = temp[f'roll_max_{window}'] / temp[f'roll_min_{window}']
temp[f'roll_abs_avg_{window}'] =(temp[f'roll_abs_max_{window}'] + temp[f'roll_abs_min_{window}'])/ 2
for i in range(4, 5):
temp[f'kstat_{i}'] = stats.kstat(z, i)
 for i in range(4, 5):
temp[f'moment_{i}'] = stats.moment(z, i)
 return temp 
<create_dataframe>
train_data['pickup_month_cos']=np.cos(( train_data['pickup_month']-1)*(2*(np.pi/12)))
 train_data['pickup_month_sin']=np.sin(( train_data['pickup_month']-1)*(2*(np.pi/12)))
 train_data['pickup_day_cos']=np.cos(( train_data['pickup_day']-1)*(2*(np.pi/30)))
 train_data['pickup_day_sin']=np.sin(( train_data['pickup_day']-1)*(2*(np.pi/30)))
 train_data['pickup_quarter_cos']=np.cos(( train_data['pickup_quarter']-1)*(2*(np.pi/4)))
 train_data['pickup_quarter_sin']=np.sin(( train_data['pickup_quarter']-1)*(2*(np.pi/4)))
 train_data['pickup_hour_cos']=np.cos(( train_data['pickup_hour']-1)*(2*(np.pi/24)))
 train_data['pickup_hour_sin']=np.sin(( train_data['pickup_hour']-1)*(2*(np.pi/24)))
 train_data.head()
def parse_sample(sample : pd.DataFrame,
batch_no : int,
window_size : List)-> pd.DataFrame:
temp = feature_enginering_by_batch(sample['signal'].values, window_size)
 temp['visual_batch'] = int(batch_no)
 return temp 
<define_variables>
train_data['pickup_year_age']=2020-train_data['pickup_year']
train_data.head()
def sample_gen(df : pd.DataFrame,
window_size : List,
batches : List=[0],)-> pd.DataFrame:
result = Parallel(n_jobs=1, temp_folder='/tmp', max_nbytes=None, backend='multiprocessing' )(delayed(parse_sample)
(df[df['visual_batch']==i], int(i), window_size)
 for i in tqdm(batches))
data = [r.values for r in result]
data = np.vstack(data)
 cols = result[0].columns
X = pd.DataFrame(data, columns=cols)
 X = reduce_mem_usage(X, False)
 X = X.sort_values('visual_batch')
 return X 
<create_dataframe>
train_data.drop(['pickup_year','pickup_quarter','pickup_month','pickup_day','pickup_hour'], axis=1, inplace=True)
 train_data.head()
def run_feat_enginnering(df : pd.DataFrame,
create_all_data_feats : bool,
window_size : List,
add_visual_batch : bool,
is_train: Optional[bool]=None)-> pd.DataFrame:
if add_visual_batching:
df = add_visual_batching(df, is_train)
 if create_all_data_feats:
df = lag_with_pct_change(df, [1, 2, 3], add_pct_change=True, add_pct_change_lag=True)
 batches = df['visual_batch'].unique().tolist()
batch_feats=sample_gen(df, window_size=window_size, batches=batches)
 df = pd.merge(df, batch_feats, on='visual_batch', how='left')
 df = reduce_mem_usage(df, False)
 return df 
<create_dataframe>
train_data.dropna(inplace=True)
 train_data = train_data[(train_data.fare_amount > 0)]
def feature_selection(df : pd.DataFrame,
df_test : pd.DataFrame,
subtract_only : Optional[bool]=True,
idx_cols : List=['time'],
target_col : List=['open_channels'])-> Tuple[pd.DataFrame , pd.DataFrame]:
drops = df.columns[df.isna().sum() >25000]
df = df.drop(drops, axis=1)
 df_test = df_test.drop(drops, axis=1)
 gc.collect()
if subtract_only == False:
corrcoef_cols = [col for col in df.columns.tolist() if col not in(idx_cols+target_col)]
first=dict() ; second=dict() ; third=dict()
for col in corrcoef_cols:
ss = np.corrcoef(df[col], df['open_channels'])[0, 1]
first[col] = ss
ss = np.corrcoef(df[col]-df['signal'], df['open_channels'])[0, 1]
second[col] = ss
ss = np.corrcoef(df[col]*df['signal'], df['open_channels'])[0, 1]
third[col] = ss
corr_df = pd.DataFrame.from_dict(
 {
'Base':first,
'Signal-Subtracted': second,
'Signal-Multiplied': third
}
).fillna(0 ).apply(np.abs ).sort_values('Base', ascending=False)
 base_cols = corr_df.sort_values('Base', ascending=False ).head(100 ).index.tolist()
multiply_cols = corr_df.sort_values('Signal-Multiplied', ascending=False ).head(10 ).index.tolist()
subtract_cols = corr_df.sort_values('Signal-Subtracted', ascending=False ).head(25 ).index.tolist()
display(corr_df.sort_values('Base', ascending=False ).tail(50))
all_cols = list(set(base_cols + multiply_cols + subtract_cols + idx_cols + target_col))
all_cols_test = list(set(base_cols + multiply_cols + subtract_cols + idx_cols))
drops = list(set(multiply_cols + subtract_cols)-set(base_cols))
df = df[all_cols]
df_test = df_test[all_cols_test]
for col in multiply_cols:
df[col+'_m'] = df[col] * df['signal']
df_test[col+'_m'] = df_test[col] * df_test['signal']
for col in subtract_cols:
df[col+'_s'] = df[col] - df['signal']
df_test[col+'_s'] = df_test[col] - df_test['signal']
df = df.drop(drops, axis=1)
 else:
not_imp = ['kstat_4', 'moment_4','signal','baseline']
subtract_cols = list(set(df.columns.tolist())-set(idx_cols + target_col + not_imp))
for col in subtract_cols:
df[col+'_s'] = df[col] - df['signal']
df_test[col+'_s'] = df_test[col] - df_test['signal']
df[col+'_b'] = df[col] - df['baseline']
df_test[col+'_b'] = df_test[col] - df_test['baseline']
df = df.replace([np.inf, -np.inf], np.nan)
 df = df.fillna(-1)
 df_test = df_test.replace([np.inf, -np.inf], np.nan)
 df_test = df_test.fillna(-1)
 df = reduce_mem_usage(df, False)
 df_test = reduce_mem_usage(df_test, False)
 gc.collect()
return df, df_test 
<compute_test_metric>
train_data = train_data[(train_data.passenger_count > 0)&(train_data.passenger_count < 10)]
def MacroF1Metric(preds : np.array, dtrain : lgb.Dataset)-> Tuple[Text, np.array, bool] :
labels = dtrain.get_label()
preds = np.round(np.clip(preds, 0, 10)).astype(int)
 score = f1_score(labels, preds, average = 'macro')
 return('MacroF1Metric', score, True )<init_hyperparams>
def distance(pickup_latitude, pickup_longitude, dropoff_latitude, dropoff_longitude):
radius = 6371
pickup_latitude, pickup_longitude, dropoff_latitude, dropoff_longitude = map(np.radians,[pickup_latitude, pickup_longitude, dropoff_latitude, dropoff_longitude])
 distance_latitude = dropoff_latitude - pickup_latitude
distance_longitude = dropoff_longitude - pickup_longitude
calculation = np.sin(distance_latitude/2.0)**2 + np.cos(pickup_latitude)* np.cos(dropoff_latitude)* np.sin(distance_longitude/2.0)**2
return 2 * radius * np.arcsin(np.sqrt(calculation))
train_data['distance'] = distance(train_data['pickup_latitude'], train_data['pickup_longitude'], train_data['dropoff_latitude'], train_data['dropoff_longitude'] )
def get_params(seed : int)-> Dict :
params = dict()
params['learning_rate']=0.1;
params['max_depth']=-1;
params['num_leaves']=128;
params['metric']='l1';
params['random_state']=seed;
params['n_jobs']=-1;
params['feature_fraction']=1 ;
params['boosting']='gbdt';
params['bagging_seed']=seed;
params['bagging_freq']=5;
params['bagging_fraction']=0.8;
params['reg_alpha']=0.1;
params['reg_lambda']=0.3
return params 
<create_dataframe>
train_data = train_data[(train_data.pickup_longitude > -80)&(train_data.pickup_longitude < -70)&(train_data.pickup_latitude > 35)&(train_data.pickup_latitude < 45)&(train_data.dropoff_longitude > -80)&(train_data.dropoff_longitude < -70)& 
(train_data.dropoff_latitude > 35)&(train_data.dropoff_latitude < 45)]
def run_cv_model_by_batch(train : pd.DataFrame,
test : pd.DataFrame,
splits : int,
shuffle : bool,
seed : int,
params : Dict,
feats : List,
sample_submission: pd.DataFrame)-> pd.DataFrame:
oof_ = np.zeros(len(train))
preds_ = np.zeros(len(test))
target = ['open_channels']
imp_df = pd.DataFrame(index=feats)
 groups = np.tile(np.arange(splits ).repeat(500000 // splits), 10)
 kf = GroupKFold(n_splits=splits)
 for n_fold,(tr_idx, val_idx)in enumerate(kf.split(train, train[target], groups=groups)) :
trn_data = lgb.Dataset(train[feats].iloc[tr_idx], label=train[target].iloc[tr_idx])
 val_data = lgb.Dataset(train[feats].iloc[val_idx], label=train[target].iloc[val_idx])
 model = lgb.train(params, trn_data, 10000, valid_sets = [trn_data, val_data], verbose_eval=0 , early_stopping_rounds = 50)
 oof_[val_idx] += model.predict(train[feats].iloc[val_idx], num_iteration=model.best_iteration)
 preds_ += model.predict(test[feats], num_iteration=model.best_iteration)/ SPLITS
f1_score_ = f1_score(train[target].iloc[val_idx], np.round(np.clip(oof_[val_idx], 0, 10)).astype(int), average = 'macro')
 rmse_score_ = np.sqrt(mean_squared_error(train[target].iloc[val_idx], oof_[val_idx]))
logger.info(f'Fold {n_fold + 1} macro f1 score : {f1_score_ :1.5f} rmse score : {rmse_score_:1.5f}')
 imp_df[f'feat_importance_{n_fold + 1}'] = model.feature_importance(importance_type='gain')
 f1_score_ = f1_score(train[target], np.round(np.clip(oof_, 0, 10)).astype(int), average = 'macro')
 rmse_score_ = np.sqrt(mean_squared_error(train[target], oof_))
logger.info(f'OOF macro f1 score : {f1_score_:1.5f} oof rmse score : {rmse_score_:1.5f}')
 sample_submission['open_channels'] = np.round(np.clip(preds_, 0, 10)).astype(int)
 sample_submission.to_csv('submission.csv', index=False, float_format='%.4f')
 display(sample_submission.head())
 np.save('oof.npy', oof_)
 np.save('preds.npy', preds_)
 return imp_df<randomize_order>
X_train, X_val, y_train, y_val = train_test_split(train_data.iloc[:, 1:], train_data['fare_amount'], test_size=0.2, random_state=42 )
def run_everything(fe_config : List)-> NoReturn:
idx_cols = ['time','batch','batch_idx','visual_batch']
target_col = ['open_channels']
type_col = ['visual_type']
init_logger()
with timer(f'Reading Data'):
logger.info('Reading Data Started...')
 train, test, sample_submission = read_data()
logger.info('Reading and Cleaning Data Completed...')
 with timer(f'Creating Features'):
logger.info('Feature Enginnering Started...')
 for config in fe_config:
train = run_feat_enginnering(train, create_all_data_feats=config[0], window_size=config[1], add_visual_batch=config[2], is_train=config[3])
 test = run_feat_enginnering(test, create_all_data_feats=config[0], window_size=config[1], add_visual_batch=config[2], is_train=config[4])
 if SELECT:
train, test = feature_selection(train, test, subtract_only=True, idx_cols=idx_cols+type_col, target_col=target_col)
 logger.info('Feature Enginnering Completed...')
 with timer(f'Running LGB model'):
logger.info(f'Training LGB model with {SPLITS} folds Started...')
 params = get_params(SEED)
 feats = [c for c in train.columns if c not in(idx_cols+target_col)]
imp = run_cv_model_by_batch(train, test, splits=SPLITS, shuffle=True, seed=SEED ,params=params, feats=feats, sample_submission=sample_submission)
 logger.info(f'Training completed...')
 return imp<categorify>
xg_boost = XGBRegressor(objective='reg:squarederror')
 xg_boost.fit(X_train,y_train)
imp = run_everything(fe_config )<set_options>
y_pred_xg_boost = xg_boost.predict(X_val)
 print('XG Boost - Root Mean Squared Error: %.2f'
% math.sqrt(mean_squared_error(y_val, y_pred_xg_boost)))
cf.go_offline()
py.init_notebook_mode()
cf.getThemes()
cf.set_config_file(theme='ggplot')
 warnings.simplefilter('ignore')
 pd.plotting.register_matplotlib_converters()
sns.mpl.rc('figure',figsize=(16, 6))
plt.style.use('ggplot')
 sns.set_style('darkgrid')
<load_from_csv>
test_data = pd.read_csv(test_path)
 test_data['pickup_datetime'] = pd.to_datetime(test_data['pickup_datetime'],format='%Y-%m-%d %H:%M:%S %Z')
 test_data['pickup_year'] = test_data['pickup_datetime'].dt.year
test_data['pickup_quarter'] = test_data['pickup_datetime'].dt.quarter
test_data['pickup_month'] = test_data['pickup_datetime'].dt.month
test_data['pickup_day'] = test_data['pickup_datetime'].dt.day
test_data['pickup_hour'] = test_data['pickup_datetime'].dt.hour
test_data['pickup_month_cos']=np.cos(( test_data['pickup_month']-1)*(2*(np.pi/12)))
 test_data['pickup_month_sin']=np.sin(( test_data['pickup_month']-1)*(2*(np.pi/12)))
 test_data['pickup_day_cos']=np.cos(( test_data['pickup_day']-1)*(2*(np.pi/30)))
 test_data['pickup_day_sin']=np.sin(( test_data['pickup_day']-1)*(2*(np.pi/30)))
 test_data['pickup_quarter_cos']=np.cos(( test_data['pickup_quarter']-1)*(2*(np.pi/4)))
 test_data['pickup_quarter_sin']=np.sin(( test_data['pickup_quarter']-1)*(2*(np.pi/4)))
 test_data['pickup_hour_cos']=np.cos(( test_data['pickup_hour']-1)*(2*(np.pi/24)))
 test_data['pickup_hour_sin']=np.sin(( test_data['pickup_hour']-1)*(2*(np.pi/24)))
 test_data['pickup_year_age']=2020-test_data['pickup_year']
test_data.drop(['pickup_year','pickup_quarter','pickup_month','pickup_day','pickup_hour','pickup_datetime'], axis=1, inplace=True)
 test_data['distance'] = distance(test_data['pickup_latitude'], test_data['pickup_longitude'], test_data['dropoff_latitude'], test_data['dropoff_longitude'])
 test_data.head()
base = os.path.abspath('/kaggle/input/liverpool-ion-switching/')
 train = pd.read_csv(os.path.join(base + '/train.csv'))
test = pd.read_csv(os.path.join(base + '/test.csv'))<feature_engineering>
y_predictions = xg_boost.predict(test_data.iloc[:, 1:] )
<define_variables><EOS> 
submission = pd.DataFrame({'key': test_data['key'], 'fare_amount': y_predictions},columns = ['key', 'fare_amount'])
 submission.to_csv('submission.csv', index = False )