Kaggle competition: New York City Taxi Fare Prediction
mean_split_df = calc_mean_importance(split_importance_list)
 plot_importance(mean_split_df, title='Model feature importance by split')
 mean_split_df = mean_split_df.reset_index().rename(columns={'index': 'feature_names'})
 mean_split_df.to_csv(outdir / 'split_importance_mean.csv', index=False )<save_to_csv>
df = pd.read_csv('/kaggle/input/new-york-city-taxi-fare-prediction/train.csv', nrows = 4000000)
 test_df = pd.read_csv('/kaggle/input/new-york-city-taxi-fare-prediction/test.csv')
 df.shape, test_df.shape
if calc_perm:
mean_perm_df = calc_mean_importance(perm_importance_list)
 plot_importance(mean_perm_df, title='Model permutation importance')
 mean_perm_df = mean_perm_df.reset_index().rename(columns={'index': 'feature_names'})
 mean_perm_df.to_csv(outdir / 'perm_importance_mean.csv', index=False )<compute_test_metric>
df.isnull().sum().sort_index() /len(df )
y_valid_pred_total_int = np.round(np.clip(y_valid_pred_total, 0, 10))
macro_f1 = macro_f1_score(y_train, y_valid_pred_total_int, 10)
 print('Total CV: macro_f1', macro_f1, 'macro_f1_list', macro_f1_list )<predict_on_test>
df.dropna(subset=['dropoff_latitude', 'dropoff_longitude'], inplace = True )
print('start model prediction...')
 with timer('preprocess test'):
X_test, _ = pp.preprocess(test, train=False)
 y_test_pred_total = np.zeros(X_test.shape[0])
 for i, model in enumerate(models):
print(f'predicting {i}-th model')
 y_pred_test = model.predict(X_test)
 y_test_pred_total += y_pred_test
y_test_pred_total /= len(models)
 y_test_pred_total_int = np.round(np.clip(y_test_pred_total, 0, 10))<save_to_csv>
df.drop(df[df['fare_amount'] < 2.5].index, axis=0, inplace = True)
 df.drop(df[df['fare_amount'] > 500].index, axis=0, inplace = True )
submission['open_channels'] = y_test_pred_total_int.astype(np.int64)
 submission.to_csv(outdir / 'submission.csv', index=False, float_format='%.4f')
 np.savez_compressed(
 outdir / 'pred.npz',
valid=y_valid_pred_total, test=y_test_pred_total)
<set_options>
df[df['passenger_count'] > 5].sort_values('passenger_count' )
sns.set_style("whitegrid")
 ROW_PER_BATCH = 500000<load_from_csv>
df.drop(df[df['pickup_longitude'] == 0].index, axis=0, inplace = True)
 df.drop(df[df['pickup_latitude'] == 0].index, axis=0, inplace = True)
 df.drop(df[df['dropoff_longitude'] == 0].index, axis=0, inplace = True)
 df.drop(df[df['dropoff_latitude'] == 0].index, axis=0, inplace = True)
 df.drop(df[df['passenger_count'] == 208].index, axis=0, inplace = True)
 df.drop(df[df['passenger_count'] > 5].index, axis=0, inplace = True)
 df.drop(df[df['passenger_count'] == 0].index, axis=0, inplace = True )
train = pd.read_csv('/kaggle/input/liverpool-ion-switching/train.csv')
 test = pd.read_csv('/kaggle/input/liverpool-ion-switching/test.csv')
 print('Shape of train is ',train.shape)
 print('Shape of test is ',test.shape )<feature_engineering>
df['key'] = pd.to_datetime(df['key'])
 key = test_df.key
test_df['key'] = pd.to_datetime(test_df['key'])
 df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'])
 test_df['pickup_datetime'] = pd.to_datetime(test_df['pickup_datetime'] )
train['batch'] = 0
for i in range(0, train.shape[0]//ROW_PER_BATCH):
train.iloc[i * ROW_PER_BATCH:(i+1)* ROW_PER_BATCH,3] = i<feature_engineering>
df['Year'] = df['pickup_datetime'].dt.year
df['Month'] = df['pickup_datetime'].dt.month
df['Date'] = df['pickup_datetime'].dt.day
df['Day of Week'] = df['pickup_datetime'].dt.dayofweek
df['Hour'] = df['pickup_datetime'].dt.hour
df.drop('pickup_datetime', axis = 1, inplace = True)
 df.drop('key', axis = 1, inplace = True)
 test_df['Year'] = test_df['pickup_datetime'].dt.year
test_df['Month'] = test_df['pickup_datetime'].dt.month
test_df['Date'] = test_df['pickup_datetime'].dt.day
test_df['Day of Week'] = test_df['pickup_datetime'].dt.dayofweek
test_df['Hour'] = test_df['pickup_datetime'].dt.hour
test_df.drop('pickup_datetime', axis = 1, inplace = True)
 test_df.drop('key', axis = 1, inplace = True )
test['batch'] = 0
for i in range(0, test.shape[0]//ROW_PER_BATCH):
test.iloc[i * ROW_PER_BATCH:(i+1)* ROW_PER_BATCH,2] = i<feature_engineering>
df.dropna(inplace=True)
 df.drop(df.index[(df.pickup_longitude < -75)| 
(df.pickup_longitude > -72)| 
(df.pickup_latitude < 40)| 
(df.pickup_latitude > 42)],inplace=True)
 df.drop(df.index[(df.dropoff_longitude < -75)| 
(df.dropoff_longitude > -72)| 
(df.dropoff_latitude < 40)| 
(df.dropoff_latitude > 42)],inplace=True )
train['type'] = 0
for i in range(train['batch'].nunique()):
median = train.iloc[i*ROW_PER_BATCH :(i+1)* ROW_PER_BATCH].signal.median()
if(median < 0):
train.iloc[i*ROW_PER_BATCH :(i+1)* ROW_PER_BATCH, train.columns.get_loc('type')] = 0
else:
train.iloc[i*ROW_PER_BATCH :(i+1)* ROW_PER_BATCH, train.columns.get_loc('type')] = 1
test['type'] = 0
ROW_PER_BATCH = 100000
for i in range(test['batch'].nunique()):
median = test.iloc[i*ROW_PER_BATCH :(i+1)*ROW_PER_BATCH].signal.median()
if(median < 0):
test.iloc[i*ROW_PER_BATCH :(i+1)* ROW_PER_BATCH, test.columns.get_loc('type')] = 0
else:
test.iloc[i*ROW_PER_BATCH :(i+1)* ROW_PER_BATCH, test.columns.get_loc('type')] = 1<feature_engineering>
def optimize_floats(df):
floats = df.select_dtypes(include=['float64'] ).columns.tolist()
df[floats] = df[floats].apply(pd.to_numeric, downcast='float')
 return df
def optimize_ints(df):
ints = df.select_dtypes(include=['int64'] ).columns.tolist()
df[ints] = df[ints].apply(pd.to_numeric, downcast='integer')
 return df
def optimize(df):
return optimize_floats(optimize_ints(df))
a = 500000; b = a *2
print('Before: mean: {} std: {} median: {}'.format(train.signal[a:b].mean() , train.signal[a:b].std() ,train.signal[a:b].median()))
a=500000; b=600000
train['signal_undrifted'] = train.signal
train.loc[train.index[a:b],'signal_undrifted'] = train.signal[a:b].values - 3*(train.time.values[a:b] - 50)/10.
 a = 500000; b = a *2
print('After: mean: {} std: {} median: {}'.format(train.signal_undrifted[a:b].mean() , train.signal_undrifted[a:b].std() ,train.signal_undrifted[a:b].median()))<feature_engineering>
df = optimize(df)
 test_df = optimize(test_df )
def f(x,low,high,mid): return -(( -low+high)/625)*(x-mid)**2+high -low
batch = 7; a = 500000*(batch-1); b = 500000*batch
train.loc[train.index[a:b],'signal_undrifted'] = train.signal.values[a:b] - f(train.time[a:b].values,-1.817,3.186,325)
 batch = 8; a = 500000*(batch-1); b = 500000*batch
train.loc[train.index[a:b],'signal_undrifted'] = train.signal.values[a:b] - f(train.time[a:b].values,-0.094,4.936,375)
 batch = 9; a = 500000*(batch-1); b = 500000*batch
train.loc[train.index[a:b],'signal_undrifted'] = train.signal.values[a:b] - f(train.time[a:b].values,1.715,6.689,425)
 batch = 10; a = 500000*(batch-1); b = 500000*batch
train.loc[train.index[a:b],'signal_undrifted'] = train.signal.values[a:b] - f(train.time[a:b].values,3.361,8.45,475 )<feature_engineering>
def calc_dists(df):
df['geodesic'] = df.apply(lambda x: geodesic_dist(x), axis = 1)
 df['circle'] = df.apply(lambda x: circle_dist(x), axis = 1)
 df['jfk'] = df.apply(lambda x: jfk_dist(x), axis = 1)
 df['lga'] = df.apply(lambda x: lga_dist(x), axis = 1)
 df['ewr'] = df.apply(lambda x: ewr_dist(x), axis = 1)
 df['tsq'] = df.apply(lambda x: tsq_dist(x), axis = 1)
 df['cpk'] = df.apply(lambda x: cpk_dist(x), axis = 1)
 df['lib'] = df.apply(lambda x: lib_dist(x), axis = 1)
 df['gct'] = df.apply(lambda x: gct_dist(x), axis = 1)
 df['met'] = df.apply(lambda x: met_dist(x), axis = 1)
 df['wtc'] = df.apply(lambda x: wtc_dist(x), axis = 1)
 return df
test['signal_undrifted'] = test.signal
start=500
a = 0; b = 100000
test.loc[test.index[a:b],'signal_undrifted'] = test.signal.values[a:b] - 3*(test.time.values[a:b]-start)/10.
 start=510
a = 100000; b = 200000
test.loc[test.index[a:b],'signal_undrifted'] = test.signal.values[a:b] - 3*(test.time.values[a:b]-start)/10.
 start=540
a = 400000; b = 500000
test.loc[test.index[a:b],'signal_undrifted'] = test.signal.values[a:b] - 3*(test.time.values[a:b]-start)/10.
 start=560
a = 600000; b = 700000
test.loc[test.index[a:b],'signal_undrifted'] = test.signal.values[a:b] - 3*(test.time.values[a:b]-start)/10.
 start=570
a = 700000; b = 800000
test.loc[test.index[a:b],'signal_undrifted'] = test.signal.values[a:b] - 3*(test.time.values[a:b]-start)/10.
 start=580
a = 800000; b = 900000
test.loc[test.index[a:b],'signal_undrifted'] = test.signal.values[a:b] - 3*(test.time.values[a:b]-start)/10.
 def f(x):
return -(0.00788)*(x-625)**2+2.345 +2.58
a = 1000000; b = 1500000
test.loc[test.index[a:b],'signal_undrifted'] = test.signal.values[a:b] - f(test.time[a:b].values )<feature_engineering>
df = calc_dists(df)
 test_df = calc_dists(test_df )
%%time
def features(df):
df = df.sort_values(by=['time'] ).reset_index(drop=True)
 df.index =(( df.time * 10_000)- 1 ).values
df['batch'] = df.index // 50_000
df['batch_index'] = df.index -(df.batch * 50_000)
 df['batch_slices'] = df['batch_index'] // 5_000
df['batch_slices2'] = df.apply(lambda r: '_'.join([str(r['batch'] ).zfill(3), str(r['batch_slices'] ).zfill(3)]), axis=1)
 for c in ['batch','batch_slices2']:
d = {}
d['mean'+c] = df.groupby([c])['signal_undrifted'].mean()
d['median'+c] = df.groupby([c])['signal_undrifted'].median()
d['max'+c] = df.groupby([c])['signal_undrifted'].max()
d['min'+c] = df.groupby([c])['signal_undrifted'].min()
d['std'+c] = df.groupby([c])['signal_undrifted'].std()
d['mean_abs_chg'+c] = df.groupby([c])['signal_undrifted'].apply(lambda x: np.mean(np.abs(np.diff(x))))
d['abs_max'+c] = df.groupby([c])['signal_undrifted'].apply(lambda x: np.max(np.abs(x)))
 d['abs_min'+c] = df.groupby([c])['signal_undrifted'].apply(lambda x: np.min(np.abs(x)))
 for v in d:
df[v] = df[c].map(d[v].to_dict())
 df['range'+c] = df['max'+c] - df['min'+c]
df['maxtomin'+c] = df['max'+c] / df['min'+c]
df['abs_avg'+c] =(df['abs_min'+c] + df['abs_max'+c])/ 2
df['signal_shift_+1'] = [0,] + list(df['signal_undrifted'].values[:-1])
 df['signal_shift_-1'] = list(df['signal_undrifted'].values[1:])+ [0]
for i in df[df['batch_index']==0].index:
df['signal_shift_+1'][i] = np.nan
for i in df[df['batch_index']==49999].index:
df['signal_shift_-1'][i] = np.nan
for c in [c1 for c1 in df.columns if c1 not in ['time', 'signal_undrifted', 'open_channels', 'batch', 'batch_index', 'batch_slices', 'batch_slices2']]:
df[c+'_msignal'] = df[c] - df['signal_undrifted']
return df
train = features(train)
 test = features(test )<compute_train_metric>
df = optimize(df)
 test_df = optimize(test_df )
def f1_score_calc(y_true, y_pred):
return f1_score(y_true, y_pred, average="macro")
 def lgb_Metric(preds, dtrain):
labels = dtrain.get_label()
preds = np.round(np.clip(preds, 0, 10)).astype(int)
 score = f1_score(labels, preds, average="macro")
 return('KaggleMetric', score, True)
 def train_model_classification(X, X_test, y, params, model_type='lgb', eval_metric='f1score',
columns=None, plot_feature_importance=False, model=None,
verbose=50, early_stopping_rounds=200, n_estimators=2000):
columns = X.columns if columns == None else columns
X_test = X_test[columns]
metrics_dict = {
'f1score': {'lgb_metric_name': lgb_Metric,}
}
result_dict = {}
oof = np.zeros(len(X))
prediction = np.zeros(( len(X_test)))
 scores = []
feature_importance = pd.DataFrame()

 if True:
X_train, X_valid, y_train, y_valid = model_selection.train_test_split(X, y, test_size=0.3, random_state=7)
 if model_type == 'lgb':
model = lgb.train(params, lgb.Dataset(X_train, y_train),
n_estimators, lgb.Dataset(X_valid, y_valid),
verbose_eval=verbose, early_stopping_rounds=early_stopping_rounds, feval=lgb_Metric)
 preds = model.predict(X_valid, num_iteration=model.best_iteration)
 y_pred_valid = np.round(np.clip(preds, 0, 10)).astype(int)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration)
 y_pred = np.round(np.clip(y_pred, 0, 10)).astype(int)
 scores.append(f1_score_calc(y_valid, y_pred_valid))
prediction += y_pred
if model_type == 'lgb' and plot_feature_importance:
fold_importance = pd.DataFrame()
fold_importance["feature"] = columns
fold_importance["importance"] = model.feature_importances_
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 print('FINAL score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 result_dict['oof'] = oof
result_dict['prediction'] = prediction
result_dict['scores'] = scores
result_dict['model'] = model
if model_type == 'lgb':
if plot_feature_importance:
feature_importance["importance"] /= folds.n_splits
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
result_dict['feature_importance'] = feature_importance
return result_dict<prepare_x_and_y>
X, y = df.drop('fare_amount', axis = 1), df['fare_amount']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=12 )
good_columns = [c for c in train.columns if c not in ['time', 'signal','open_channels', 'batch', 'batch_index', 'batch_slices', 'batch_slices2']]
X = train[good_columns].copy()
y = train['open_channels']
X_test = test[good_columns].copy()<drop_column>
dtrain = xgb.DMatrix(X_train, label=y_train)
 dvalid = xgb.DMatrix(X_test, label=y_test)
 dtest = xgb.DMatrix(test_df)
 watchlist = [(dtrain, 'train'),(dvalid, 'valid')]
xgb_params = {
'min_child_weight': 1,
'learning_rate': 0.05,
'colsample_bytree': 0.7,
'max_depth': 10,
'subsample': 0.7,
'n_estimators': 5000,
'n_jobs': -1,
'booster' : 'gbtree',
'silent': 1,
'eval_metric': 'rmse'}
model = xgb.train(xgb_params, dtrain, 700, watchlist, early_stopping_rounds=100, maximize=False, verbose_eval=50 )
del train, test<train_model>
y_train_pred = model.predict(dtrain)
 y_pred = model.predict(dvalid)
 print('Train r2 score: ', r2_score(y_train_pred, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred))
train_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train))
test_rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print(f'Train RMSE: {train_rmse:.4f}')
 print(f'Test RMSE: {test_rmse:.4f}' )
params = {'learning_rate': 0.1, 'max_depth': 7, 'num_leaves':2**7+1, 'metric': 'rmse', 'random_state': 7, 'n_jobs':-1}
result_dict_lgb = train_model_classification(X=X[0:500000*8-1], X_test=X_test, y=y[0:500000*8-1], params=params, model_type='lgb', eval_metric='f1score', plot_feature_importance=False,
verbose=50, early_stopping_rounds=150, n_estimators=3000 )<save_to_csv>
test_preds = model.predict(dtest )
sub = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv')
 sub['open_channels'] = np.array(np.round(result_dict_lgb['prediction'],0), np.int)
 sub.to_csv('submission_unshifted_70p.csv', index=False, float_format='%.4f')
 sub.head(10 )<import_modules>
test_preds = model.predict(dtest)
 submission = pd.DataFrame(
 {'key': key, 'fare_amount': test_preds},
columns = ['key', 'fare_amount'])
 submission.to_csv('submission1.csv', index = False )