Kaggle competition: New York City Taxi Trip Duration
<SOS> metric: RMSLE Kaggle data source: new-york-city-taxi-trip-duration<load_from_csv>
import os
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import seaborn as sns
import math
from datetime import datetime
%%time
train = pd.read_csv('.. /input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32} )<train_on_grid>
FILEPATH = os.path.join(".. ", "input")
 TRAINPATH = os.path.join(FILEPATH, "train.csv")
 TESTPATH = os.path.join(FILEPATH, "test.csv" )
rows = 150_000
segments = int(np.floor(train.shape[0] / rows))
def add_trend_feature(arr, abs_values=False):
idx = np.array(range(len(arr)))
 if abs_values:
arr = np.abs(arr)
 lr = LinearRegression()
lr.fit(idx.reshape(-1, 1), arr)
 return lr.coef_[0]
def classic_sta_lta(x, length_sta, length_lta):
sta = np.cumsum(x ** 2)
 sta = np.require(sta, dtype=np.float)
 lta = sta.copy()
sta[length_sta:] = sta[length_sta:] - sta[:-length_sta]
sta /= length_sta
lta[length_lta:] = lta[length_lta:] - lta[:-length_lta]
lta /= length_lta
sta[:length_lta - 1] = 0
dtiny = np.finfo(0.0 ).tiny
idx = lta < dtiny
lta[idx] = dtiny
return sta / lta
X_tr = pd.DataFrame(index=range(segments), dtype=np.float64)
 y_tr = pd.DataFrame(index=range(segments), dtype=np.float64, columns=['time_to_failure'])
 total_mean = train['acoustic_data'].mean()
total_std = train['acoustic_data'].std()
total_max = train['acoustic_data'].max()
total_min = train['acoustic_data'].min()
total_sum = train['acoustic_data'].sum()
total_abs_sum = np.abs(train['acoustic_data'] ).sum()
def calc_change_rate(x):
change =(np.diff(x)/ x[:-1] ).values
change = change[np.nonzero(change)[0]]
change = change[~np.isnan(change)]
change = change[change != -np.inf]
change = change[change != np.inf]
return np.mean(change)
 for segment in tqdm_notebook(range(segments)) :
seg = train.iloc[segment*rows:segment*rows+rows]
x = pd.Series(seg['acoustic_data'].values)
 y = seg['time_to_failure'].values[-1]
y_tr.loc[segment, 'time_to_failure'] = y
X_tr.loc[segment, 'mean'] = x.mean()
X_tr.loc[segment, 'std'] = x.std()
X_tr.loc[segment, 'max'] = x.max()
X_tr.loc[segment, 'min'] = x.min()
X_tr.loc[segment, 'mean_change_abs'] = np.mean(np.diff(x))
X_tr.loc[segment, 'mean_change_rate'] = calc_change_rate(x)
 X_tr.loc[segment, 'abs_max'] = np.abs(x ).max()
X_tr.loc[segment, 'abs_min'] = np.abs(x ).min()
X_tr.loc[segment, 'std_first_50000'] = x[:50000].std()
X_tr.loc[segment, 'std_last_50000'] = x[-50000:].std()
X_tr.loc[segment, 'std_first_10000'] = x[:10000].std()
X_tr.loc[segment, 'std_last_10000'] = x[-10000:].std()
X_tr.loc[segment, 'avg_first_50000'] = x[:50000].mean()
X_tr.loc[segment, 'avg_last_50000'] = x[-50000:].mean()
X_tr.loc[segment, 'avg_first_10000'] = x[:10000].mean()
X_tr.loc[segment, 'avg_last_10000'] = x[-10000:].mean()
X_tr.loc[segment, 'min_first_50000'] = x[:50000].min()
X_tr.loc[segment, 'min_last_50000'] = x[-50000:].min()
X_tr.loc[segment, 'min_first_10000'] = x[:10000].min()
X_tr.loc[segment, 'min_last_10000'] = x[-10000:].min()
X_tr.loc[segment, 'max_first_50000'] = x[:50000].max()
X_tr.loc[segment, 'max_last_50000'] = x[-50000:].max()
X_tr.loc[segment, 'max_first_10000'] = x[:10000].max()
X_tr.loc[segment, 'max_last_10000'] = x[-10000:].max()
X_tr.loc[segment, 'max_to_min'] = x.max() / np.abs(x.min())
 X_tr.loc[segment, 'max_to_min_diff'] = x.max() - np.abs(x.min())
 X_tr.loc[segment, 'count_big'] = len(x[np.abs(x)> 500])
 X_tr.loc[segment, 'sum'] = x.sum()
X_tr.loc[segment, 'mean_change_rate_first_50000'] = calc_change_rate(x[:50000])
 X_tr.loc[segment, 'mean_change_rate_last_50000'] = calc_change_rate(x[-50000:])
 X_tr.loc[segment, 'mean_change_rate_first_10000'] = calc_change_rate(x[:10000])
 X_tr.loc[segment, 'mean_change_rate_last_10000'] = calc_change_rate(x[-10000:])
 X_tr.loc[segment, 'q95'] = np.quantile(x, 0.95)
 X_tr.loc[segment, 'q99'] = np.quantile(x, 0.99)
 X_tr.loc[segment, 'q05'] = np.quantile(x, 0.05)
 X_tr.loc[segment, 'q01'] = np.quantile(x, 0.01)
 X_tr.loc[segment, 'abs_q95'] = np.quantile(np.abs(x), 0.95)
 X_tr.loc[segment, 'abs_q99'] = np.quantile(np.abs(x), 0.99)
 X_tr.loc[segment, 'abs_q05'] = np.quantile(np.abs(x), 0.05)
 X_tr.loc[segment, 'abs_q01'] = np.quantile(np.abs(x), 0.01)
 X_tr.loc[segment, 'trend'] = add_trend_feature(x)
 X_tr.loc[segment, 'abs_trend'] = add_trend_feature(x, abs_values=True)
 X_tr.loc[segment, 'abs_mean'] = np.abs(x ).mean()
X_tr.loc[segment, 'abs_std'] = np.abs(x ).std()
X_tr.loc[segment, 'mad'] = x.mad()
X_tr.loc[segment, 'kurt'] = x.kurtosis()
X_tr.loc[segment, 'skew'] = x.skew()
X_tr.loc[segment, 'med'] = x.median()
X_tr.loc[segment, 'Hilbert_mean'] = np.abs(hilbert(x)).mean()
X_tr.loc[segment, 'Hann_window_mean'] =(convolve(x, hann(150), mode='same')/ sum(hann(150)) ).mean()
X_tr.loc[segment, 'classic_sta_lta1_mean'] = classic_sta_lta(x, 500, 10000 ).mean()
X_tr.loc[segment, 'classic_sta_lta2_mean'] = classic_sta_lta(x, 5000, 100000 ).mean()
X_tr.loc[segment, 'classic_sta_lta3_mean'] = classic_sta_lta(x, 3333, 6666 ).mean()
X_tr.loc[segment, 'classic_sta_lta4_mean'] = classic_sta_lta(x, 10000, 25000 ).mean()
X_tr.loc[segment, 'classic_sta_lta5_mean'] = classic_sta_lta(x, 50, 1000 ).mean()
X_tr.loc[segment, 'classic_sta_lta6_mean'] = classic_sta_lta(x, 100, 5000 ).mean()
X_tr.loc[segment, 'classic_sta_lta7_mean'] = classic_sta_lta(x, 333, 666 ).mean()
X_tr.loc[segment, 'classic_sta_lta8_mean'] = classic_sta_lta(x, 4000, 10000 ).mean()
X_tr.loc[segment, 'Moving_average_700_mean'] = x.rolling(window=700 ).mean().mean(skipna=True)
 ewma = pd.Series.ewm
X_tr.loc[segment, 'exp_Moving_average_300_mean'] =(ewma(x, span=300 ).mean() ).mean(skipna=True)
 X_tr.loc[segment, 'exp_Moving_average_3000_mean'] = ewma(x, span=3000 ).mean().mean(skipna=True)
 X_tr.loc[segment, 'exp_Moving_average_30000_mean'] = ewma(x, span=30000 ).mean().mean(skipna=True)
 no_of_std = 3
X_tr.loc[segment, 'MA_700MA_std_mean'] = x.rolling(window=700 ).std().mean()
X_tr.loc[segment,'MA_700MA_BB_high_mean'] =(X_tr.loc[segment, 'Moving_average_700_mean'] + no_of_std * X_tr.loc[segment, 'MA_700MA_std_mean'] ).mean()
X_tr.loc[segment,'MA_700MA_BB_low_mean'] =(X_tr.loc[segment, 'Moving_average_700_mean'] - no_of_std * X_tr.loc[segment, 'MA_700MA_std_mean'] ).mean()
X_tr.loc[segment, 'MA_400MA_std_mean'] = x.rolling(window=400 ).std().mean()
X_tr.loc[segment,'MA_400MA_BB_high_mean'] =(X_tr.loc[segment, 'Moving_average_700_mean'] + no_of_std * X_tr.loc[segment, 'MA_400MA_std_mean'] ).mean()
X_tr.loc[segment,'MA_400MA_BB_low_mean'] =(X_tr.loc[segment, 'Moving_average_700_mean'] - no_of_std * X_tr.loc[segment, 'MA_400MA_std_mean'] ).mean()
X_tr.loc[segment, 'MA_1000MA_std_mean'] = x.rolling(window=1000 ).std().mean()
X_tr.drop('Moving_average_700_mean', axis=1, inplace=True)
 X_tr.loc[segment, 'iqr'] = np.subtract(*np.percentile(x, [75, 25]))
X_tr.loc[segment, 'q999'] = np.quantile(x,0.999)
 X_tr.loc[segment, 'q001'] = np.quantile(x,0.001)
 X_tr.loc[segment, 'ave10'] = stats.trim_mean(x, 0.1)
 for windows in [10, 100, 1000]:
x_roll_std = x.rolling(windows ).std().dropna().values
x_roll_mean = x.rolling(windows ).mean().dropna().values
X_tr.loc[segment, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X_tr.loc[segment, 'std_roll_std_' + str(windows)] = x_roll_std.std()
X_tr.loc[segment, 'max_roll_std_' + str(windows)] = x_roll_std.max()
X_tr.loc[segment, 'min_roll_std_' + str(windows)] = x_roll_std.min()
X_tr.loc[segment, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)
 X_tr.loc[segment, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)
 X_tr.loc[segment, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)
 X_tr.loc[segment, 'q99_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.99)
 X_tr.loc[segment, 'av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))
X_tr.loc[segment, 'av_change_rate_roll_std_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 X_tr.loc[segment, 'abs_max_roll_std_' + str(windows)] = np.abs(x_roll_std ).max()
X_tr.loc[segment, 'ave_roll_mean_' + str(windows)] = x_roll_mean.mean()
X_tr.loc[segment, 'std_roll_mean_' + str(windows)] = x_roll_mean.std()
X_tr.loc[segment, 'max_roll_mean_' + str(windows)] = x_roll_mean.max()
X_tr.loc[segment, 'min_roll_mean_' + str(windows)] = x_roll_mean.min()
X_tr.loc[segment, 'q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.01)
 X_tr.loc[segment, 'q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.05)
 X_tr.loc[segment, 'q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.95)
 X_tr.loc[segment, 'q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.99)
 X_tr.loc[segment, 'av_change_abs_roll_mean_' + str(windows)] = np.mean(np.diff(x_roll_mean))
X_tr.loc[segment, 'av_change_rate_roll_mean_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 X_tr.loc[segment, 'abs_max_roll_mean_' + str(windows)] = np.abs(x_roll_mean ).max()<compute_test_metric>
df = pd.read_csv(TRAINPATH, index_col=0)
 df.head()
np.abs(X_tr.corrwith(y_tr['time_to_failure'])).sort_values(ascending=False ).head(12 )<feature_engineering>
df_test = pd.read_csv(TESTPATH)
 df_test.head()
means_dict = {}
for col in X_tr.columns:
if X_tr[col].isnull().any() :
print(col)
 mean_value = X_tr.loc[X_tr[col] != -np.inf, col].mean()
X_tr.loc[X_tr[col] == -np.inf, col] = mean_value
X_tr[col] = X_tr[col].fillna(mean_value)
 means_dict[col] = mean_value<train_model>
df = df[(df['trip_duration'] > 60)&(df['trip_duration'] < 3600 * 6)]
scaler = StandardScaler()
scaler.fit(X_tr)
 X_train_scaled = pd.DataFrame(scaler.transform(X_tr), columns=X_tr.columns )<load_from_csv>
df['store_and_fwd_flag'] = df['store_and_fwd_flag'].astype('category' ).cat.codes
df_test['store_and_fwd_flag'] = df_test['store_and_fwd_flag'].astype('category' ).cat.codes
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 X_test = pd.DataFrame(columns=X_tr.columns, dtype=np.float64, index=submission.index)
 plt.figure(figsize=(22, 16))
for i, seg_id in enumerate(tqdm_notebook(X_test.index)) :
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 x = pd.Series(seg['acoustic_data'].values)
 X_test.loc[seg_id, 'mean'] = x.mean()
X_test.loc[seg_id, 'std'] = x.std()
X_test.loc[seg_id, 'max'] = x.max()
X_test.loc[seg_id, 'min'] = x.min()
X_test.loc[seg_id, 'mean_change_abs'] = np.mean(np.diff(x))
X_test.loc[seg_id, 'mean_change_rate'] = calc_change_rate(x)
 X_test.loc[seg_id, 'abs_max'] = np.abs(x ).max()
X_test.loc[seg_id, 'abs_min'] = np.abs(x ).min()
X_test.loc[seg_id, 'std_first_50000'] = x[:50000].std()
X_test.loc[seg_id, 'std_last_50000'] = x[-50000:].std()
X_test.loc[seg_id, 'std_first_10000'] = x[:10000].std()
X_test.loc[seg_id, 'std_last_10000'] = x[-10000:].std()
X_test.loc[seg_id, 'avg_first_50000'] = x[:50000].mean()
X_test.loc[seg_id, 'avg_last_50000'] = x[-50000:].mean()
X_test.loc[seg_id, 'avg_first_10000'] = x[:10000].mean()
X_test.loc[seg_id, 'avg_last_10000'] = x[-10000:].mean()
X_test.loc[seg_id, 'min_first_50000'] = x[:50000].min()
X_test.loc[seg_id, 'min_last_50000'] = x[-50000:].min()
X_test.loc[seg_id, 'min_first_10000'] = x[:10000].min()
X_test.loc[seg_id, 'min_last_10000'] = x[-10000:].min()
X_test.loc[seg_id, 'max_first_50000'] = x[:50000].max()
X_test.loc[seg_id, 'max_last_50000'] = x[-50000:].max()
X_test.loc[seg_id, 'max_first_10000'] = x[:10000].max()
X_test.loc[seg_id, 'max_last_10000'] = x[-10000:].max()
X_test.loc[seg_id, 'max_to_min'] = x.max() / np.abs(x.min())
 X_test.loc[seg_id, 'max_to_min_diff'] = x.max() - np.abs(x.min())
 X_test.loc[seg_id, 'count_big'] = len(x[np.abs(x)> 500])
 X_test.loc[seg_id, 'sum'] = x.sum()
X_test.loc[seg_id, 'mean_change_rate_first_50000'] = calc_change_rate(x[:50000])
 X_test.loc[seg_id, 'mean_change_rate_last_50000'] = calc_change_rate(x[-50000:])
 X_test.loc[seg_id, 'mean_change_rate_first_10000'] = calc_change_rate(x[:10000])
 X_test.loc[seg_id, 'mean_change_rate_last_10000'] = calc_change_rate(x[-10000:])
 X_test.loc[seg_id, 'q95'] = np.quantile(x,0.95)
 X_test.loc[seg_id, 'q99'] = np.quantile(x,0.99)
 X_test.loc[seg_id, 'q05'] = np.quantile(x,0.05)
 X_test.loc[seg_id, 'q01'] = np.quantile(x,0.01)
 X_test.loc[seg_id, 'abs_q95'] = np.quantile(np.abs(x), 0.95)
 X_test.loc[seg_id, 'abs_q99'] = np.quantile(np.abs(x), 0.99)
 X_test.loc[seg_id, 'abs_q05'] = np.quantile(np.abs(x), 0.05)
 X_test.loc[seg_id, 'abs_q01'] = np.quantile(np.abs(x), 0.01)
 X_test.loc[seg_id, 'trend'] = add_trend_feature(x)
 X_test.loc[seg_id, 'abs_trend'] = add_trend_feature(x, abs_values=True)
 X_test.loc[seg_id, 'abs_mean'] = np.abs(x ).mean()
X_test.loc[seg_id, 'abs_std'] = np.abs(x ).std()
X_test.loc[seg_id, 'mad'] = x.mad()
X_test.loc[seg_id, 'kurt'] = x.kurtosis()
X_test.loc[seg_id, 'skew'] = x.skew()
X_test.loc[seg_id, 'med'] = x.median()
X_test.loc[seg_id, 'Hilbert_mean'] = np.abs(hilbert(x)).mean()
X_test.loc[seg_id, 'Hann_window_mean'] =(convolve(x, hann(150), mode='same')/ sum(hann(150)) ).mean()
X_test.loc[seg_id, 'classic_sta_lta1_mean'] = classic_sta_lta(x, 500, 10000 ).mean()
X_test.loc[seg_id, 'classic_sta_lta2_mean'] = classic_sta_lta(x, 5000, 100000 ).mean()
X_test.loc[seg_id, 'classic_sta_lta3_mean'] = classic_sta_lta(x, 3333, 6666 ).mean()
X_test.loc[seg_id, 'classic_sta_lta4_mean'] = classic_sta_lta(x, 10000, 25000 ).mean()
X_test.loc[seg_id, 'classic_sta_lta5_mean'] = classic_sta_lta(x, 50, 1000 ).mean()
X_test.loc[seg_id, 'classic_sta_lta6_mean'] = classic_sta_lta(x, 100, 5000 ).mean()
X_test.loc[seg_id, 'classic_sta_lta7_mean'] = classic_sta_lta(x, 333, 666 ).mean()
X_test.loc[seg_id, 'classic_sta_lta8_mean'] = classic_sta_lta(x, 4000, 10000 ).mean()
X_test.loc[seg_id, 'Moving_average_700_mean'] = x.rolling(window=700 ).mean().mean(skipna=True)
 ewma = pd.Series.ewm
X_test.loc[seg_id, 'exp_Moving_average_300_mean'] =(ewma(x, span=300 ).mean() ).mean(skipna=True)
 X_test.loc[seg_id, 'exp_Moving_average_3000_mean'] = ewma(x, span=3000 ).mean().mean(skipna=True)
 X_test.loc[seg_id, 'exp_Moving_average_30000_mean'] = ewma(x, span=30000 ).mean().mean(skipna=True)
 no_of_std = 3
X_test.loc[seg_id, 'MA_700MA_std_mean'] = x.rolling(window=700 ).std().mean()
X_test.loc[seg_id,'MA_700MA_BB_high_mean'] =(X_test.loc[seg_id, 'Moving_average_700_mean'] + no_of_std * X_test.loc[seg_id, 'MA_700MA_std_mean'] ).mean()
X_test.loc[seg_id,'MA_700MA_BB_low_mean'] =(X_test.loc[seg_id, 'Moving_average_700_mean'] - no_of_std * X_test.loc[seg_id, 'MA_700MA_std_mean'] ).mean()
X_test.loc[seg_id, 'MA_400MA_std_mean'] = x.rolling(window=400 ).std().mean()
X_test.loc[seg_id,'MA_400MA_BB_high_mean'] =(X_test.loc[seg_id, 'Moving_average_700_mean'] + no_of_std * X_test.loc[seg_id, 'MA_400MA_std_mean'] ).mean()
X_test.loc[seg_id,'MA_400MA_BB_low_mean'] =(X_test.loc[seg_id, 'Moving_average_700_mean'] - no_of_std * X_test.loc[seg_id, 'MA_400MA_std_mean'] ).mean()
X_test.loc[seg_id, 'MA_1000MA_std_mean'] = x.rolling(window=1000 ).std().mean()
X_test.drop('Moving_average_700_mean', axis=1, inplace=True)
 X_test.loc[seg_id, 'iqr'] = np.subtract(*np.percentile(x, [75, 25]))
X_test.loc[seg_id, 'q999'] = np.quantile(x,0.999)
 X_test.loc[seg_id, 'q001'] = np.quantile(x,0.001)
 X_test.loc[seg_id, 'ave10'] = stats.trim_mean(x, 0.1)
 for windows in [10, 100, 1000]:
x_roll_std = x.rolling(windows ).std().dropna().values
x_roll_mean = x.rolling(windows ).mean().dropna().values
X_test.loc[seg_id, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X_test.loc[seg_id, 'std_roll_std_' + str(windows)] = x_roll_std.std()
X_test.loc[seg_id, 'max_roll_std_' + str(windows)] = x_roll_std.max()
X_test.loc[seg_id, 'min_roll_std_' + str(windows)] = x_roll_std.min()
X_test.loc[seg_id, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)
 X_test.loc[seg_id, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)
 X_test.loc[seg_id, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)
 X_test.loc[seg_id, 'q99_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.99)
 X_test.loc[seg_id, 'av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))
X_test.loc[seg_id, 'av_change_rate_roll_std_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 X_test.loc[seg_id, 'abs_max_roll_std_' + str(windows)] = np.abs(x_roll_std ).max()
X_test.loc[seg_id, 'ave_roll_mean_' + str(windows)] = x_roll_mean.mean()
X_test.loc[seg_id, 'std_roll_mean_' + str(windows)] = x_roll_mean.std()
X_test.loc[seg_id, 'max_roll_mean_' + str(windows)] = x_roll_mean.max()
X_test.loc[seg_id, 'min_roll_mean_' + str(windows)] = x_roll_mean.min()
X_test.loc[seg_id, 'q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.01)
 X_test.loc[seg_id, 'q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.05)
 X_test.loc[seg_id, 'q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.95)
 X_test.loc[seg_id, 'q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.99)
 X_test.loc[seg_id, 'av_change_abs_roll_mean_' + str(windows)] = np.mean(np.diff(x_roll_mean))
X_test.loc[seg_id, 'av_change_rate_roll_mean_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 X_test.loc[seg_id, 'abs_max_roll_mean_' + str(windows)] = np.abs(x_roll_mean ).max()
if i < 12:
plt.subplot(6, 4, i + 1)
 plt.plot(seg['acoustic_data'])
 plt.title(seg_id)
 for col in X_test.columns:
if X_test[col].isnull().any() :
X_test.loc[X_test[col] == -np.inf, col] = means_dict[col]
X_test[col] = X_test[col].fillna(means_dict[col])
 X_test_scaled = pd.DataFrame(scaler.transform(X_test), columns=X_test.columns )<choose_model_class>
from sklearn.decomposition import PCA
n_fold = 5
folds = KFold(n_splits=n_fold, shuffle=True, random_state=11 )<split>
coords = np.vstack(( df[['pickup_latitude', 'pickup_longitude']].values,
df[['dropoff_latitude', 'dropoff_longitude']].values,
df_test[['pickup_latitude', 'pickup_longitude']].values,
df_test[['dropoff_latitude', 'dropoff_longitude']].values))
pca = PCA().fit(coords)
 df['pickup_pca0'] = pca.transform(df[['pickup_latitude', 'pickup_longitude']])[:, 0]
df['pickup_pca1'] = pca.transform(df[['pickup_latitude', 'pickup_longitude']])[:, 1]
df['dropoff_pca0'] = pca.transform(df[['dropoff_latitude', 'dropoff_longitude']])[:, 0]
df['dropoff_pca1'] = pca.transform(df[['dropoff_latitude', 'dropoff_longitude']])[:, 1]
df_test['pickup_pca0'] = pca.transform(df_test[['pickup_latitude', 'pickup_longitude']])[:, 0]
df_test['pickup_pca1'] = pca.transform(df_test[['pickup_latitude', 'pickup_longitude']])[:, 1]
df_test['dropoff_pca0'] = pca.transform(df_test[['dropoff_latitude', 'dropoff_longitude']])[:, 0]
df_test['dropoff_pca1'] = pca.transform(df_test[['dropoff_latitude', 'dropoff_longitude']])[:, 1]
def train_model(X=X_train_scaled, X_test=X_test_scaled, y=y_tr, params=None, folds=folds, model_type='lgb', plot_feature_importance=False, model=None):
oof = np.zeros(len(X))
prediction = np.zeros(len(X_test))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
if model_type == 'lgb':
model = lgb.LGBMRegressor(**params, n_estimators = 50000, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='mae',
verbose=10000, early_stopping_rounds=200)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration_)
 if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train, feature_names=X.columns)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X.columns)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid ).reshape(-1,)
 score = mean_absolute_error(y_valid, y_pred_valid)
 print(f'Fold {fold_n}.MAE: {score:.4f}.')
 print('')
 y_pred = model.predict(X_test ).reshape(-1,)
 if model_type == 'cat':
model = CatBoostRegressor(iterations=20000, eval_metric='MAE', **params)
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 oof[valid_index] = y_pred_valid.reshape(-1,)
 scores.append(mean_absolute_error(y_valid, y_pred_valid))
prediction += y_pred
if model_type == 'lgb':
fold_importance = pd.DataFrame()
fold_importance["feature"] = X.columns
fold_importance["importance"] = model.feature_importances_
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_fold
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 if model_type == 'lgb':
feature_importance["importance"] /= n_fold
if plot_feature_importance:
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
return oof, prediction, feature_importance
return oof, prediction
else:
return oof, prediction<init_hyperparams>
df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'])
 df['dropoff_datetime'] = pd.to_datetime(df['dropoff_datetime'])
 df_test['pickup_datetime'] = pd.to_datetime(df_test['pickup_datetime'] )
params = {'num_leaves': 128,
'min_data_in_leaf': 79,
'objective': 'huber',
'max_depth': -1,
'learning_rate': 0.01,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": 0.8126672064208567,
"bagging_seed": 11,
"metric": 'mae',
"verbosity": -1,
'reg_alpha': 0.1302650970728192,
'reg_lambda': 0.3603427518866501
}
oof_lgb, prediction_lgb, feature_importance = train_model(params=params, model_type='lgb', plot_feature_importance=True )<groupby>
df['hour'] = df.pickup_datetime.dt.hour
df['day'] = df.pickup_datetime.dt.dayofweek
df['month'] = df.pickup_datetime.dt.month
df_test['hour'] = df_test.pickup_datetime.dt.hour
df_test['day'] = df_test.pickup_datetime.dt.dayofweek
df_test['month'] = df_test.pickup_datetime.dt.month
top_cols = list(feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index )<train_model>
df['distance2'] = np.sqrt(( df['pickup_pca0']-df['dropoff_pca0'])**2
+(df['pickup_pca1']-df['dropoff_pca1'])**2)
 df_test['distance2'] = np.sqrt(( df_test['pickup_pca0']-df_test['dropoff_pca0'])**2
+(df_test['pickup_pca1']-df_test['dropoff_pca1'])**2 )
oof_lgb, prediction_lgb, feature_importance = train_model(X=X_train_scaled, X_test=X_test_scaled, params=params, model_type='lgb', plot_feature_importance=True )<train_model>
df['log_trip_duration'] = np.log(df['trip_duration'] )
xgb_params = {'eta': 0.03,
'max_depth': 9,
'subsample': 0.85,
'objective': 'reg:linear',
'eval_metric': 'mae',
'silent': True,
'nthread': 4}
oof_xgb, prediction_xgb = train_model(X=X_train_scaled, X_test=X_test_scaled, params=xgb_params, model_type='xgb' )<train_model>
NUM_VARS = ['pickup_pca0', 'pickup_pca1', 'dropoff_pca0', 'dropoff_pca1', 'month', 'hour', 'day', 'distance2']
TARGET = 'log_trip_duration'
model = NuSVR(gamma='scale', nu=0.9, C=10.0, tol=0.01)
 oof_svr, prediction_svr = train_model(X=X_train_scaled, X_test=X_test_scaled, params=None, model_type='sklearn', model=model )<train_model>
num_features = NUM_VARS
model = NuSVR(gamma='scale', nu=0.7, tol=0.01, C=1.0)
 oof_svr1, prediction_svr1 = train_model(X=X_train_scaled, X_test=X_test_scaled, params=None, model_type='sklearn', model=model )<train_model>
X_train = df.loc[:, num_features]
y_train = df[TARGET]
X_test = df_test.loc[:, num_features]
X_train.shape, y_train.shape, X_test.shape
params = {'loss_function':'MAE'}
oof_cat, prediction_cat = train_model(X=X_train_scaled, X_test=X_test_scaled, params=params, model_type='cat' )<train_model>
from sklearn.ensemble import RandomForestRegressor
model = KernelRidge(kernel='rbf', alpha=0.1, gamma=0.01)
 oof_r, prediction_r = train_model(X=X_train_scaled, X_test=X_test_scaled, params=None, model_type='sklearn', model=model )<create_dataframe>
from sklearn.ensemble import RandomForestRegressor
train_stack = np.vstack([oof_lgb, oof_xgb, oof_svr, oof_svr1, oof_r, oof_cat] ).transpose()
train_stack = pd.DataFrame(train_stack, columns = ['lgb', 'xgb', 'svr', 'svr1', 'r', 'cat'])
 test_stack = np.vstack([prediction_lgb, prediction_xgb, prediction_svr, prediction_svr1, prediction_r, prediction_cat] ).transpose()
test_stack = pd.DataFrame(test_stack )<train_model>
m = RandomForestRegressor(n_estimators=17, min_samples_leaf=10, min_samples_split=15, max_features='auto', max_depth=80, bootstrap=True)
 m.fit(X_train, y_train )
oof_lgb_stack, prediction_lgb_stack, feature_importance = train_model(X=train_stack, X_test=test_stack, params=params, model_type='lgb', plot_feature_importance=True )<save_to_csv>
from sklearn.model_selection import cross_val_score
submission['time_to_failure'] =(prediction_lgb + prediction_xgb + prediction_svr + prediction_svr1 + prediction_cat + prediction_r)/ 6
print(submission.head())
 submission.to_csv('submission.csv' )<save_to_csv>
cv_scores = cross_val_score(m, X_train, y_train, cv=5, scoring='neg_mean_squared_log_error')
 cv_scores
X_tr.to_csv('train_features.csv', index=False)
 X_test.to_csv('test_features.csv', index=False )<import_modules>
y_test_pred = m.predict(X_test)
 y_test_pred[:5]
import matplotlib.pyplot as plt
from tqdm import tqdm
from sklearn.preprocessing import StandardScaler
from sklearn.svm import NuSVR
from sklearn.kernel_ridge import KernelRidge
from sklearn.metrics import mean_absolute_error, make_scorer
from sklearn.model_selection import GridSearchCV<load_from_csv>
my_submission = pd.DataFrame({'id': df_test.id, 'trip_duration': np.exp(y_test_pred)})
 my_submission.to_csv('submission.csv', index=False )