Kaggle competition: New York City Taxi Fare Prediction
warnings.simplefilter('ignore')
 warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 1000)
 pd.set_option('display.max_rows', 500)
 %matplotlib inline<define_search_space>
df=pd.read_csv('/kaggle/input/new-york-city-taxi-fare-prediction/train.csv',parse_dates=['pickup_datetime'], nrows=500000)
 df.dropna(inplace=True )
BATCHSIZE = 50000
SEED = 529
SELECT = True
SPLITS = 5
fe_config = [ 
(True, True, 50000, None), 
(False, False, 5000, None),
]<define_variables>
test=pd.read_csv('/kaggle/input/new-york-city-taxi-fare-prediction/test.csv',parse_dates=['pickup_datetime'])
 test
def seed_everything(seed : int)-> NoReturn :
rn.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 seed_everything(SEED)
<load_from_csv>
nyc_min_longitude = -74.3
nyc_max_longitude = -72
nyc_min_latitude = 40.63
nyc_max_latitude = 42
def read_data(base : os.path.abspath)-> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
train = pd.read_csv(os.path.join(base + '/train.csv'), dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int8})
 test = pd.read_csv(os.path.join(base + '/test.csv'), dtype={'time': np.float32, 'signal': np.float32})
 sub = pd.read_csv(os.path.join(base + '/sample_submission.csv'), dtype={'time': np.float32})
 return train, test, sub 
<create_dataframe>
for long in ['pickup_longitude', 'dropoff_longitude']:
df = df[(df[long] > nyc_min_longitude)&(df[long] < nyc_max_longitude)]
for lat in ['pickup_latitude', 'dropoff_latitude']:
df = df[(df[lat] > nyc_min_latitude)&(df[lat] < nyc_max_latitude)]
def batching(df : pd.DataFrame,
batch_size : int,
add_index : Optional[bool]=True)-> pd.DataFrame :
df['batch_'+ str(batch_size)] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values + 1
df['batch_'+ str(batch_size)] = df['batch_'+ str(batch_size)].astype(np.uint16)
 if add_index:
df['batch_' + str(batch_size)+'_idx'] = df.index -(df['batch_'+ str(batch_size)] * batch_size)
 df['batch_' + str(batch_size)+'_idx'] = df['batch_' + str(batch_size)+'_idx'].astype(np.uint16)
 return df 
<compute_test_metric>
df.loc[df['passenger_count'] == 0, 'passenger_count'] = 1
def maddest(d : Union[np.array, pd.Series, List], axis : Optional[int]=None)-> np.array:
return np.mean(np.absolute(d - np.mean(d, axis)) , axis)
<normalization>
df = df[(df['fare_amount'] >0)&(df['fare_amount'] <= 100)]
def denoise_signal(x : Union[np.array, pd.Series],
wavelet : Optional[Text]='db4',
level : Optional[int]=1)-> np.array:
coeff = pywt.wavedec(x, wavelet, mode="per")
 sigma =(1/0.6745)* maddest(coeff[-level])
 uthresh = sigma * np.sqrt(2*np.log(len(x)))
 coeff[1:] =(pywt.threshold(i, value=uthresh, mode='hard')for i in coeff[1:])
 return pywt.waverec(coeff, wavelet, mode='per')
<normalization>
def euc_distance(lat1, long1, lat2, long2):
return(((lat1-lat2)**2 +(long1-long2)**2)**0.5)
 df['travel_distance'] = euc_distance(df['pickup_latitude'], df['pickup_longitude'], df['dropoff_latitude'], df['dropoff_longitude'])
 test['travel_distance'] = euc_distance(test['pickup_latitude'], test['pickup_longitude'], test['dropoff_latitude'], test['dropoff_longitude'] )
def denoise_signal_simple(x : Union[np.array, pd.Series],
wavelet : Optional[Text]='db4',
level : Optional[int]=1)-> np.array:
coeff = pywt.wavedec(x, wavelet, mode="per")
 uthresh = 10
coeff[1:] =(pywt.threshold(i, value=uthresh, mode='hard')for i in coeff[1:])
 return pywt.waverec(coeff, wavelet, mode='per')
<train_on_grid>
df['year'] = df['pickup_datetime'].dt.year
df['month'] = df['pickup_datetime'].dt.month
df['day'] = df['pickup_datetime'].dt.day
df['day_of_week'] = df['pickup_datetime'].dt.dayofweek
df['hour'] = df['pickup_datetime'].dt.hour
df = df.drop(['pickup_datetime'], axis=1 )
def trend(df : Union[pd.Series, np.array],
abs_values: Optional[bool]=False)-> float:
idx = np.array(range(len(df)))
 if abs_values:
df = np.abs(df)
 lr = LinearRegression()
lr.fit(idx.reshape(-1, 1), df)
 return lr.coef_[0] 
<feature_engineering>
test['year'] = test['pickup_datetime'].dt.year
test['month'] =test['pickup_datetime'].dt.month
test['day'] = test['pickup_datetime'].dt.day
test['day_of_week'] =test['pickup_datetime'].dt.dayofweek
test['hour'] = test['pickup_datetime'].dt.hour
test = test.drop(['pickup_datetime'], axis=1 )
def change_rate(df : Union[pd.Series, np.array])-> float:
change =(np.diff(df)/ df[:-1])
 change = change[np.nonzero(change)[0]]
change = change[~np.isnan(change)]
change = change[change != -np.inf]
change = change[change != np.inf]
return np.mean(change)
<create_dataframe>
test.drop(['key'], axis=1,inplace=True)
 df.drop(['key'], axis=1,inplace=True )
def lag_with_pct_change(df : pd.DataFrame,
batch_size : int,
shift_sizes : Optional[List]=[1, 2],
add_pct_change : Optional[bool]=False,
add_pct_change_lag : Optional[bool]=False)-> pd.DataFrame:
assert 'batch_' + str(batch_size)+'_idx' in df.columns
for shift_size in shift_sizes:
df['signal_shift_pos_'+str(shift_size)] = df['signal'].shift(shift_size ).fillna(0)
 df['signal_shift_neg_'+str(shift_size)] = df['signal'].shift(-1*shift_size ).fillna(0)
 for i in df[df['batch_' + str(batch_size)+'_idx'].isin(range(shift_size)) ].index:
df['signal_shift_pos_'+str(shift_size)][i] = np.nan
for i in df[df['batch_' + str(batch_size)+'_idx'].isin(range(batch_size - shift_size, batch_size)) ].index:
df['signal_shift_neg_'+str(shift_size)][i] = np.nan
if add_pct_change:
df['pct_change'] = df['signal'].pct_change()
if add_pct_change_lag:
df['pct_change_shift_pos_'+str(shift_size)] = df['pct_change'].shift(shift_size ).fillna(0)
 df['pct_change_shift_neg_'+str(shift_size)] = df['pct_change'].shift(-1*shift_size ).fillna(0)
 for i in df[df['batch_' + str(batch_size)+'_idx'].isin(range(shift_size)) ].index:
df['pct_change_shift_pos_'+str(shift_size)][i] = np.nan
for i in df[df['batch_' + str(batch_size)+'_idx'].isin(range(batch_size - shift_size, batch_size)) ].index:
df['pct_change_shift_neg_'+str(shift_size)][i] = np.nan
return df 
<feature_engineering>
print(df.isnull().sum())
 print(test.isnull().sum() )
def feature_enginering_by_batch(z : Union[pd.Series, np.array],
batch_size : int,
window_size : Optional[List]=None)-> pd.DataFrame:
temp = pd.DataFrame(index=[0], dtype=np.float16)
 temp['mean'] = z.mean()
temp['max'] = z.max()
temp['min'] = z.min()
temp['std'] = z.std()
temp['mean_abs_chg'] = np.mean(np.abs(np.diff(z)))
 temp['abs_max'] = np.max(np.abs(z))
temp['abs_min'] = np.min(np.abs(z))
temp['range'] = temp['max'] - temp['min']
temp['max_to_min'] = temp['max'] / temp['min']
temp['abs_avg'] =(temp['abs_max'] + temp['abs_min'])/ 2
for i in range(1, 5):
temp[f'kstat_{i}'] = stats.kstat(z, i)
 for i in range(2, 5):
temp[f'moment_{i}'] = stats.moment(z, i)
 for i in [1, 2]:
temp[f'kstatvar_{i}'] = stats.kstatvar(z, i)
 if window_size is not None:
for window in window_size:
temp['percentile_roll_'+str(window)+'_std_25'] = np.percentile(pd.Series(z ).rolling(window ).std().dropna().values, 25)
 temp['percentile_roll_'+str(window)+'_std_75'] = np.percentile(pd.Series(z ).rolling(window ).std().dropna().values, 75)
 temp['percentile_roll_'+str(window)+'_std_05'] = np.percentile(pd.Series(z ).rolling(window ).std().dropna().values, 5)
 temp['percentile_roll_'+str(window)+'_std_95'] = np.percentile(pd.Series(z ).rolling(window ).std().dropna().values, 95)
 temp['percentile_roll_'+str(window)+'_mean_25'] = np.percentile(pd.Series(z ).rolling(window ).mean().dropna().values, 25)
 temp['percentile_roll_'+str(window)+'_mean_75'] = np.percentile(pd.Series(z ).rolling(window ).mean().dropna().values, 75)
 temp['percentile_roll_'+str(window)+'_mean_05'] = np.percentile(pd.Series(z ).rolling(window ).mean().dropna().values, 5)
 temp['percentile_roll_'+str(window)+'_mean_95'] = np.percentile(pd.Series(z ).rolling(window ).mean().dropna().values, 95)
 return temp 
<create_dataframe>
import pandas as pd
import numpy as np
from sklearn.preprocessing import scale
from sklearn.model_selection import train_test_split
from keras.models import Sequential
from keras.layers import Dense
from sklearn.metrics import mean_squared_error
def parse_sample(sample : pd.DataFrame,
batch_no : int,
batch_size : int,
window_size : List)-> pd.DataFrame:
temp = feature_enginering_by_batch(sample['signal'].values, batch_size, window_size)
 temp['batch_'+ str(batch_size)] = int(batch_no)
 return temp 
<define_variables>
df_prescaled = df.copy()
df_scaled = df.drop(['fare_amount'], axis=1)
 df_scaled = scale(df_scaled)
 cols = df.columns.tolist()
cols.remove('fare_amount')
 df_scaled = pd.DataFrame(df_scaled, columns=cols, index=df.index)
 df_scaled = pd.concat([df_scaled, df['fare_amount']], axis=1)
 df = df_scaled.copy()
def sample_gen(df : pd.DataFrame,
batch_size : int,
window_size : List,
batches : List=[0],)-> pd.DataFrame:
result = Parallel(n_jobs=1, temp_folder='/tmp', max_nbytes=None, backend='multiprocessing' )(delayed(parse_sample)
(df[df['batch_'+ str(batch_size)]==i], int(i), batch_size, window_size)
 for i in tqdm(batches))
data = [r.values for r in result]
data = np.vstack(data)
 cols = result[0].columns
cols = [name+'_'+str(batch_size)if name!='batch_'+ str(batch_size)else 'batch_'+ str(batch_size)for name in cols ]
X = pd.DataFrame(data, columns=cols)
 X = reduce_mem_usage(X, False)
 X = X.sort_values('batch_'+ str(batch_size))
return X 
<create_dataframe>
X = df.loc[:, df.columns != 'fare_amount']
y = df.fare_amount
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2 )
def run_feat_enginnering(df : pd.DataFrame,
create_all_data_feats : bool,
add_index : bool,
batch_size : int,
window_size : List)-> pd.DataFrame:
df = batching(df, batch_size=batch_size, add_index=add_index)
 if create_all_data_feats:
df = lag_with_pct_change(df, batch_size, [1, 2, 4], add_pct_change=True, add_pct_change_lag=True)
 batches = df['batch_'+ str(batch_size)].unique().tolist()
batch_feats=sample_gen(df, batch_size=batch_size, window_size=window_size, batches=batches)
 df = pd.merge(df, batch_feats, on='batch_'+ str(batch_size), how='left')
 df = reduce_mem_usage(df, False)
 return df 
<create_dataframe>
model=Sequential()
model.add(Dense(128, activation= 'relu', input_dim=X_train.shape[1]))
model.add(Dense(64, activation= 'relu'))
model.add(Dense(32, activation= 'relu'))
model.add(Dense(8, activation= 'relu'))
model.add(Dense(1))
model.compile(loss='mse', optimizer='adam', metrics=['mse'])
 model.fit(X_train, y_train, epochs=16)
 train_pred = model.predict(X_train)
 train_rmse = np.sqrt(mean_squared_error(y_train, train_pred))
test_pred = model.predict(X_test)
 test_rmse = np.sqrt(mean_squared_error(y_test, test_pred))
print("Train RMSE: {:0.2f}".format(train_rmse))
print("Test RMSE: {:0.2f}".format(test_rmse))
print('------------------------' )
def feature_selection(df : pd.DataFrame,
df_test : pd.DataFrame,
subtract_only : Optional[bool]=True,
idx_cols : List=['time'],
target_col : List=['open_channels'])-> Tuple[pd.DataFrame , pd.DataFrame]:
drops = df.columns[df.isna().sum() >25000]
df = df.drop(drops, axis=1)
 df = df.replace([np.inf, -np.inf], np.nan)
 df = df.fillna(0)
 gc.collect()
if subtract_only == False:
corrcoef_cols = [col for col in df.columns.tolist() if col not in(idx_cols+target_col)]
first=dict() ; second=dict() ; third=dict()
for col in corrcoef_cols:
ss = np.corrcoef(df[col], df['open_channels'])[0, 1]
first[col] = ss
ss = np.corrcoef(df[col]-df['signal'], df['open_channels'])[0, 1]
second[col] = ss
ss = np.corrcoef(df[col]*df['signal'], df['open_channels'])[0, 1]
third[col] = ss
corr_df = pd.DataFrame.from_dict(
 {
'Base':first,
'Signal-Subtracted': second,
'Signal-Multiplied': third
}
).fillna(0 ).apply(np.abs ).sort_values('Base', ascending=False)
 base_cols = corr_df.sort_values('Base', ascending=False ).head(100 ).index.tolist()
multiply_cols = corr_df.sort_values('Signal-Multiplied', ascending=False ).head(10 ).index.tolist()
subtract_cols = corr_df.sort_values('Signal-Subtracted', ascending=False ).head(25 ).index.tolist()
display(corr_df.sort_values('Base', ascending=False ).tail(50))
all_cols = list(set(base_cols + multiply_cols + subtract_cols + idx_cols + target_col))
all_cols_test = list(set(base_cols + multiply_cols + subtract_cols + idx_cols))
drops = list(set(multiply_cols + subtract_cols)-set(base_cols))
df = df[all_cols]
df_test = df_test[all_cols_test]
for col in multiply_cols:
df[col+'_m'] = df[col] * df['signal']
df_test[col+'_m'] = df_test[col] * df_test['signal']
for col in subtract_cols:
df[col+'_s'] = df[col] - df['signal']
df_test[col+'_s'] = df_test[col] - df_test['signal']
df = df.drop(drops, axis=1)
 else:
not_imp = ['kstat_1_5000', 'kstat_2_5000', 'kstat_3_5000', 'kstat_4_5000', 'moment_2_5000',
'moment_3_5000','moment_4_5000', 'kstatvar_1_5000', 'kstatvar_2_5000','kstat_1_50000',
'kstat_2_50000', 'kstat_3_50000', 'kstat_4_50000', 'moment_2_50000', 'moment_3_50000',
'moment_4_50000', 'kstatvar_1_50000', 'kstatvar_2_50000']
subtract_cols = list(set(df.columns.tolist())-set(idx_cols + target_col + not_imp))
for col in subtract_cols:
df[col+'_s'] = df[col] - df['signal']
df_test[col+'_s'] = df_test[col] - df_test['signal']
df = reduce_mem_usage(df, False)
 df_test = reduce_mem_usage(df_test, False)
 gc.collect()
return df, df_test 
<compute_test_metric>
test_scaled = scale(test )
def MacroF1Metric(preds : np.array, dtrain : lgb.Dataset)-> Tuple[Text, np.array, bool] :
labels = dtrain.get_label()
preds = np.round(np.clip(preds, 0, 10)).astype(int)
 score = f1_score(labels, preds, average = 'macro')
 return('MacroF1Metric', score, True)
<create_dataframe>
pred= model.predict(test_scaled)
 pred.shape
<init_hyperparams><EOS> 
submission = pd.read_csv('/kaggle/input/new-york-city-taxi-fare-prediction/sample_submission.csv')
 submission['fare_amount'] = pred
submission.to_csv('submission.csv',index=False )