Kaggle competition: New York City Taxi Trip Duration
<SOS> metric: RMSLE Kaggle data source: new-york-city-taxi-trip-duration<categorify>
%matplotlib inline
test_leak = rewrite_compiled_leak(test_leak, best_lag)
 test_leak[['ID']+leaky_cols+['compiled_leak']].head()<save_to_csv>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
test_res = test_leak[leaky_cols+['compiled_leak']].replace(0.0, np.nan)
 test_res.to_csv('test_leak.csv', index=False )<feature_engineering>
train.isna().sum()
test_leak.loc[test_leak["compiled_leak"]==0, "compiled_leak"] = test_leak.loc[test_leak["compiled_leak"]==0, "nonzero_mean"]<save_to_csv>
train.duplicated().sum()
sub = test[["ID"]]
sub["target"] = test_leak["compiled_leak"]
sub.to_csv(f"baseline_sub_lag_{best_lag}.csv", index=False)
 print(f"baseline_sub_lag_{best_lag}.csv saved" )<set_options>
train = train[train['passenger_count']>0]
train = train[train['passenger_count']<6]
%matplotlib inline<load_from_csv>
train = train.loc[train['pickup_longitude']> -85]
train = train.loc[train['pickup_latitude']< 46]
%%time
data = pd.read_csv('.. /input/santander-value-prediction-challenge/train.csv')
 target = np.log1p(data['target'])
 data.drop(['ID', 'target'], axis=1, inplace=True )<load_from_csv>
train = train.loc[train['dropoff_longitude']> -80]
train = train.loc[train['dropoff_latitude']> 36]
%%time
leak = pd.read_csv('.. /input/breaking-lb-fresh-start-with-lag-selection/train_leak.csv')
 data['leak'] = leak['compiled_leak'].values
data['log_leak'] = np.log1p(leak['compiled_leak'].values )<train_on_grid>
train = train[(train['trip_duration'] > 60)&(train['trip_duration'] < 3600)]
train[train['trip_duration']<=120].shape
train[train['trip_duration']>=3600].shape
train['trip_duration'] = np.log(train['trip_duration'].values)
 train['hour'] = train['pickup_datetime'].apply(lambda x: int(x.split() [1][0:2]))
test['hour'] = test['pickup_datetime'].apply(lambda x: int(x.split() [1][0:2]))
%%time
def rmse(y_true, y_pred):
return mean_squared_error(y_true, y_pred)**.5
reg = XGBRegressor(n_estimators=1000)
 folds = KFold(4, True, 134259)
 fold_idx = [(trn_, val_)for trn_, val_ in folds.split(data)]
scores = []
nb_values = data.nunique(dropna=False)
 nb_zeros =(data == 0 ).astype(np.uint8 ).sum(axis=0)
 features = [f for f in data.columns if f not in ['log_leak', 'leak', 'target', 'ID']]
for _f in features:
score = 0
for trn_, val_ in fold_idx:
reg.fit(
 data[['log_leak', _f]].iloc[trn_], target.iloc[trn_],
eval_set=[(data[['log_leak', _f]].iloc[val_], target.iloc[val_])],
eval_metric='rmse',
early_stopping_rounds=50,
verbose=False 
)
 score += rmse(target.iloc[val_], reg.predict(data[['log_leak', _f]].iloc[val_], ntree_limit=reg.best_ntree_limit)) / folds.n_splits
scores.append(( _f, score))<create_dataframe>
def haversine(lat1, lon1, lat2, lon2):
R = 6372800
phi1, phi2 = math.radians(lat1), math.radians(lat2)
 dphi = math.radians(lat2 - lat1)
 dlambda = math.radians(lon2 - lon1)
 a = math.sin(dphi/2)**2 + \
math.cos(phi1)*math.cos(phi2)*math.sin(dlambda/2)**2
return 2*R*math.atan2(math.sqrt(a), math.sqrt(1 - a))
train['dist_long'] = train['pickup_longitude'] - train['dropoff_longitude']
test['dist_long'] = test['pickup_longitude'] - test['dropoff_longitude']
train['dist_lat'] = train['pickup_latitude'] - train['dropoff_latitude']
test['dist_lat'] = test['pickup_latitude'] - test['dropoff_latitude']
train['dist'] = np.sqrt(np.square(train['dist_long'])+ np.square(train['dist_lat']))
test['dist'] = np.sqrt(np.square(test['dist_long'])+ np.square(test['dist_lat']))
report = pd.DataFrame(scores, columns=['feature', 'rmse'] ).set_index('feature')
 report['nb_zeros'] = nb_zeros
report['nunique'] = nb_values
report.sort_values(by='rmse', ascending=True, inplace=True )<prepare_x_and_y>
train['pickup_datetime'] = pd.to_datetime(train['pickup_datetime'])
 train['dropoff_datetime'] = pd.to_datetime(train['dropoff_datetime'])
 test['pickup_datetime'] = pd.to_datetime(test['pickup_datetime'] )
report.sort_values('rmse', ascending=False, inplace=True)
 radii = 1000 *(report['rmse'].max() - report['rmse'] ).values
source = ColumnDataSource(data=dict(
 x=report['nunique'].tolist() ,
y=report['nb_zeros'].tolist() ,
desc=report.index.tolist() ,
radius=radii,
fill_color=[
"
],
rmse=report['rmse'].tolist() 
))
TOOLTIPS = [ 
("rmse", "@rmse"), 
("(nunique, nb_zeros)", "(@x, @y)"), 
("feature", "@desc"),
]
TOOLS = "hover, crosshair, pan, wheel_zoom, zoom_in, zoom_out, box_zoom, undo, redo, reset, tap, save, box_select, poly_select, lasso_select"
p = figure(plot_width=600, plot_height=600, tooltips=TOOLTIPS, tools=TOOLS,
title="Number of unique values vs Number of zeros")
 p.xaxis.axis_label = 'Number of unique values in feature'
p.yaxis.axis_label = 'Number of zeros in feature'
p.xaxis.axis_label_text_font_style ='bold'
p.yaxis.axis_label_text_font_style ='bold'
p.title.text_color = '
p.title.text_font_size = '16pt'
p.scatter(
 'x', 'y', source=source,
radius='radius',
fill_color='fill_color',
line_color=None,
fill_alpha=0.8 
)
 output_notebook()
show(p )<save_to_csv>
train['minute'] = train.pickup_datetime.dt.minute
train['hour'] = train.pickup_datetime.dt.hour
train['day'] = train.pickup_datetime.dt.dayofweek
train['month'] = train.pickup_datetime.dt.month
test['minute'] = train.pickup_datetime.dt.minute
test['hour'] = test.pickup_datetime.dt.hour
test['day'] = test.pickup_datetime.dt.dayofweek
test['month'] = test.pickup_datetime.dt.month
report.to_csv('feature_report.csv', index=True )<filter>
y = train["trip_duration"]
X_train = train[["vendor_id","passenger_count","pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","dist","month","hour","day"]]
X_test = test[["vendor_id","passenger_count","pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","dist","month","hour","day"]]
good_features = report.loc[report['rmse'] <= 0.7925].index
rmses = report.loc[report['rmse'] <= 0.7925, 'rmse'].values
good_features<load_from_csv>
from sklearn.model_selection import train_test_split, cross_val_score
test = pd.read_csv('.. /input/santander-value-prediction-challenge/test.csv' )<load_from_csv>
xtrain, xvalid, ytrain, yvalid = train_test_split(X_train,y, test_size=0.2, random_state=42)
 xtrain.shape, xvalid.shape, xtrain.shape, yvalid.shape
%%time
tst_leak = pd.read_csv('.. /input/breaking-lb-fresh-start-with-lag-selection/test_leak.csv')
 test['leak'] = tst_leak['compiled_leak']
test['log_leak'] = np.log1p(tst_leak['compiled_leak'] )<feature_engineering>
import lightgbm as lgb
folds = KFold(n_splits=5, shuffle=True, random_state=1)
 features = [f for f in data if f not in ['ID', 'leak', 'log_leak', 'target']]
data.replace(0, np.nan, inplace=True)
 data['log_of_mean'] = np.log1p(data[features].replace(0, np.nan ).mean(axis=1))
data['mean_of_log'] = np.log1p(data[features] ).replace(0, np.nan ).mean(axis=1)
 data['log_of_median'] = np.log1p(data[features].replace(0, np.nan ).median(axis=1))
data['nb_nans'] = data[features].isnull().sum(axis=1)
 data['the_sum'] = np.log1p(data[features].sum(axis=1))
data['the_std'] = data[features].std(axis=1)
 data['the_kur'] = data[features].kurtosis(axis=1)
 test.replace(0, np.nan, inplace=True)
 test['log_of_mean'] = np.log1p(test[features].replace(0, np.nan ).mean(axis=1))
test['mean_of_log'] = np.log1p(test[features] ).replace(0, np.nan ).mean(axis=1)
 test['log_of_median'] = np.log1p(test[features].replace(0, np.nan ).median(axis=1))
test['nb_nans'] = test[features].isnull().sum(axis=1)
 test['the_sum'] = np.log1p(test[features].sum(axis=1))
test['the_std'] = test[features].std(axis=1)
 test['the_kur'] = test[features].kurtosis(axis=1)
 features = good_features.tolist()
features = features + ['log_leak', 'log_of_mean', 'mean_of_log', 'log_of_median', 'nb_nans', 'the_sum', 'the_std', 'the_kur']
dtrain = lgb.Dataset(data=data[features],
label=target, free_raw_data=False)
 test['target'] = 0
dtrain.construct()
oof_preds = np.zeros(data.shape[0])
 for trn_idx, val_idx in folds.split(data):
lgb_params = {
'objective': 'regression',
'num_leaves': 58,
'subsample': 0.6143,
'colsample_bytree': 0.6453,
'min_split_gain': np.power(10, -2.5988),
'reg_alpha': np.power(10, -2.2887),
'reg_lambda': np.power(10, 1.7570),
'min_child_weight': np.power(10, -0.1477),
'verbose': -1,
'seed': 3,
'boosting_type': 'gbdt',
'max_depth': -1,
'learning_rate': 0.05,
'metric': 'l2',
}
clf = lgb.train(
 params=lgb_params,
train_set=dtrain.subset(trn_idx),
valid_sets=dtrain.subset(val_idx),
num_boost_round=10000,
early_stopping_rounds=100,
verbose_eval=0 
)
 oof_preds[val_idx] = clf.predict(dtrain.data.iloc[val_idx])
 test['target'] += clf.predict(test[features])/ folds.n_splits
print(mean_squared_error(target.iloc[val_idx],
oof_preds[val_idx])**.5)
 data['predictions'] = oof_preds
data.loc[data['leak'].notnull() , 'predictions'] = np.log1p(data.loc[data['leak'].notnull() , 'leak'])
 print('OOF SCORE : %9.6f'
%(mean_squared_error(target, oof_preds)**.5))
print('OOF SCORE with LEAK : %9.6f'
%(mean_squared_error(target, data['predictions'])**.5)) 
<save_to_csv>
dtrain = lgb.Dataset(X_train,y )
test['target'] = np.expm1(test['target'])
 test.loc[test['leak'].notnull() , 'target'] = test.loc[test['leak'].notnull() , 'leak']
test[['ID', 'target']].to_csv('leaky_submission.csv', index=False, float_format='%.2f' )<set_options>
lgb_params = {
'learning_rate': 0.1,
'max_depth': 25,
'num_leaves': 1000,
'objective': 'regression',
'feature_fraction': 0.9,
'bagging_fraction': 0.5,
'max_bin': 1000}
warnings.filterwarnings('ignore')
<import_modules>
model_lgb = lgb.train(lgb_params,
dtrain,
num_boost_round=1200)
import torch
import torch.nn as nn
import torch.optim as optim
from torch.autograd import Variable<load_from_csv>
cv_results = lgb.cv(
 lgb_params,
dtrain,
num_boost_round=100,
nfold=3,
metrics='mae',
early_stopping_rounds=10,
stratified=False
)
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )<prepare_x_and_y>
print('Current parameters:
', lgb_params)
 print('
Best num_boost_round:', len(cv_results['l1-mean']))
print('Best CV score:', cv_results['l1-mean'][-1] )
y = np.log1p(train_df.target.values )<drop_column>
pred_test = np.exp(model_lgb.predict(X_test))
pred_test
train_df.drop(["ID", 'target'], axis=1, inplace = True)
 test_df.drop(["ID"], axis=1, inplace = True )<concatenate>
submit = pd.read_csv('.. /input/sample_submission.csv' )
df = pd.concat([train_df, test_df] ).reset_index(drop=True)
 del train_df, test_df
gc.collect() ;<feature_engineering>
submit['trip_duration'] = pred_test
submit.head()
df.loc[:, :] = np.log1p(df.values )<define_variables>
submit_file = pd.DataFrame({"id": test.id, "trip_duration": pred_test} )
SEED = 717
np.random.seed = SEED<normalization>
submit_file.to_csv('submission.csv', index=False )