Kaggle competition: New York City Taxi Trip Duration
<SOS> metric: RMSLE Kaggle data source: new-york-city-taxi-trip-duration<create_dataframe>
warnings.filterwarnings("ignore")
 %matplotlib inline
<find_best_params>
TRAINPATH = '.. /input/ny-taxis/train_w_zones2.csv'
TESTPATH = '.. /input/ny-taxis/test_w_zones2.csv'
df_train = pd.read_csv(TRAINPATH)
 df_test = pd.read_csv(TESTPATH)
 print('df_train:', df_train.shape, '
df_test:', df_test.shape)
 l=df_train.shape[0]
df_train.head()
env = nflrush.make_env()
param=train_my_model(train_df)
 CDF=stats.norminvgauss.cdf(np.linspace(-99,99,199),param[0],param[1],param[2],param[3])
 for(test_df, sample_prediction_df)in env.iter_test() :
predictions_df = pd.DataFrame(data=[CDF], columns= sample_prediction_df.columns)
 env.predict(predictions_df)
 env.write_submission_file()
print([filename for filename in os.listdir('/kaggle/working')if '.csv' in filename] )<load_from_csv>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 transact_cols = [f for f in train.columns if f not in ["ID", "target"]]
y = np.log1p(train["target"] ).values<feature_engineering>
test["target"] = train["target"].mean()<define_variables>
cols = ['f190486d6', '58e2e02e6', 'eeb9cd3aa', '9fd594eec', '6eef030c1', '15ace8c9f',
'fb0f5dbfe', '58e056e12', '20aa07010', '024c577b9', 'd6bb78916', 'b43a7cfd5',
'58232a6fb', '1702b5bf0', '324921c7b', '62e59a501', '2ec5b290f', '241f0f867',
'fb49e4212', '66ace2992', 'f74e8f13d', '5c6487af1', '963a49cdc', '26fc93eb7',
'1931ccfdd', '703885424', '70feb1494', '491b9ee45', '23310aa6f', 'e176a204a',
'6619d81fc', '1db387535',
'fc99f9426', '91f701ba2', '0572565c2', '190db8488', 'adb64ff71', 'c47340d97', 'c5a231d81', '0ff32eb98'
]<categorify>
def _get_leak(df, cols, lag=0):
d1 = df[cols[:-lag-2]].apply(tuple, axis=1 ).to_frame().rename(columns={0: 'key'})
 d2 = df[cols[lag+2:]].apply(tuple, axis=1 ).to_frame().rename(columns={0: 'key'})
 d2['pred'] = df[cols[lag]]
d3 = d2[~d2.duplicated(['key'], keep=False)]
return d1.merge(d3, how='left', on='key' ).pred.fillna(0 )<feature_engineering>
def compiled_leak_result() :
max_nlags = len(cols)- 2
train_leak = train[["ID", "target"] + cols]
train_leak["compiled_leak"] = 0
train_leak["nonzero_mean"] = train[transact_cols].apply(
 lambda x: np.expm1(np.log1p(x[x!=0] ).mean()), axis=1 
)
 scores = []
leaky_value_counts = []
leaky_value_corrects = []
leaky_cols = []
for i in range(max_nlags):
c = "leaked_target_"+str(i)
 print('Processing lag', i)
 train_leak[c] = _get_leak(train_leak, cols, i)
 leaky_cols.append(c)
 train_leak = train.join(
 train_leak.set_index("ID")[leaky_cols+["compiled_leak", "nonzero_mean"]],
on="ID", how="left" 
)[["ID", "target"] + cols + leaky_cols+["compiled_leak", "nonzero_mean"]]
zeroleak = train_leak["compiled_leak"]==0
train_leak.loc[zeroleak, "compiled_leak"] = train_leak.loc[zeroleak, c]
leaky_value_counts.append(sum(train_leak["compiled_leak"] > 0))
_correct_counts = sum(train_leak["compiled_leak"]==train_leak["target"])
 leaky_value_corrects.append(_correct_counts/leaky_value_counts[-1])
 print("Leak values found in train", leaky_value_counts[-1])
 print(
 "% of correct leaks values in train ",
leaky_value_corrects[-1] 
)
 tmp = train_leak.copy()
tmp.loc[zeroleak, "compiled_leak"] = tmp.loc[zeroleak, "nonzero_mean"]
scores.append(np.sqrt(mean_squared_error(y, np.log1p(tmp["compiled_leak"] ).fillna(14.49))))
print(
 'Score(filled with nonzero mean)',
scores[-1] 
)
 result = dict(
 score=scores,
leaky_count=leaky_value_counts,
leaky_correct=leaky_value_corrects, 
)
 return train_leak, result<create_dataframe>
def featuresSelection(df_in):
VARS_CAT = [ 'store_and_fwd_flag' ]
VARS_NUM = ['pickup_latitude', 'pickup_longitude', 'dropoff_latitude', 'dropoff_longitude', 'zone', 'distances', 'pickup_Month', 'pickup_Hour', 'pickup_Weekend', 'passenger_count', 'vendor_id' ]
vars_cat = VARS_CAT
vars_num = VARS_NUM
X=df_in.loc[:, vars_cat + vars_num]
for cat in vars_cat:
X[cat] = X[cat].astype('category' ).cat.codes
return X
result = pd.DataFrame.from_dict(result, orient='columns')
 result.T<save_to_csv>
from sklearn.model_selection import cross_val_score, train_test_split
from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import SGDRegressor
from sklearn.metrics import mean_squared_log_error as MSLE
import xgboost
result.to_csv('train_leaky_stat.csv', index=False )<compute_test_metric>
X_train_sample, X_validation, y_train_sample, y_validation = train_test_split(X_train, y_train, test_size=.2, random_state=42)
 print(X_train_sample.shape, y_train_sample.shape , X_validation.shape, y_validation.shape)
 X_train_sample.head(5 )
best_score = np.min(result['score'])
 best_lag = np.argmin(result['score'])
 print('best_score', best_score, '
best_lag', best_lag )<feature_engineering>
def rewrite_compiled_leak(leak_df, lag):
leak_df["compiled_leak"] = 0
for i in range(lag):
c = "leaked_target_"+str(i)
 zeroleak = leak_df["compiled_leak"]==0
leak_df.loc[zeroleak, "compiled_leak"] = leak_df.loc[zeroleak, c]
return leak_df<categorify>
%%time
modelparams = { 'booster':'gbtree', 'verbosity':1, 'max_depth':15, 'subsample': 1, 'lamda':0, 'max_delta_step':3, 'objective':'reg:linear', 'learning_rate':0.08, 'colsample_bytree':0.9, 'colsample_bylevel':0.9}
data_train = xgboost.DMatrix(X_train,y_train)
 xg = xgboost.train(modelparams, data_train, num_boost_round=200 )
leaky_cols = [c for c in train_leak.columns if 'leaked_target_' in c]
train_leak = rewrite_compiled_leak(train_leak, best_lag)
 train_leak[['ID']+leaky_cols+['compiled_leak']].head()<save_to_csv>
train_res = train_leak[leaky_cols+['compiled_leak']].replace(0.0, np.nan)
 train_res.to_csv('train_leak.csv', index=False )<feature_engineering>
X_test = featuresSelection(df_test)
 X_test.head()
def compiled_leak_result_test(max_nlags):
test_leak = test[["ID", "target"] + cols]
test_leak["compiled_leak"] = 0
test_leak["nonzero_mean"] = test[transact_cols].apply(
 lambda x: np.expm1(np.log1p(x[x!=0] ).mean()), axis=1 
)
 scores = []
leaky_value_counts = []
leaky_cols = []
for i in range(max_nlags):
c = "leaked_target_"+str(i)
 print('Processing lag', i)
 test_leak[c] = _get_leak(test_leak, cols, i)
 leaky_cols.append(c)
 test_leak = test.join(
 test_leak.set_index("ID")[leaky_cols+["compiled_leak", "nonzero_mean"]],
on="ID", how="left" 
)[["ID", "target"] + cols + leaky_cols+["compiled_leak", "nonzero_mean"]]
zeroleak = test_leak["compiled_leak"]==0
test_leak.loc[zeroleak, "compiled_leak"] = test_leak.loc[zeroleak, c]
leaky_value_counts.append(sum(test_leak["compiled_leak"] > 0))
print("Leak values found in test", leaky_value_counts[-1])
 result = dict(
 leaky_count=leaky_value_counts, 
)
 return test_leak, result<categorify>
y_test_predict = np.expm1(xg.predict(xgboost.DMatrix(X_test)))
 y_test_predict[:5]
test_leak, test_result = compiled_leak_result_test(max_nlags=38 )<create_dataframe>
submission = pd.DataFrame(df_test.loc[:, 'id'])
 submission['trip_duration']=y_test_predict
print(submission.shape)
 submission.head()
<save_to_csv><EOS> 
submission.to_csv("submit_file.csv", index=False )