Kaggle competition: New York City Taxi Trip Duration
<SOS> metric: RMSLE Kaggle data source: new-york-city-taxi-trip-duration<set_options>
sns.set({'figure.figsize':(16,10)})
 %matplotlib inline
%matplotlib inline
warnings.filterwarnings("ignore" )<load_from_csv>
train = pd.read_csv('.. /input/dataset-de-landry/train.csv')
 test = pd.read_csv('.. /input/dataset-de-landry/test.csv')
 sample = pd.read_csv('.. /input/dataset-de-landry/sample_submission.csv' )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
train.isna().sum()
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<categorify>
train = train[train['passenger_count']>0]
train = train[train['passenger_count']<9]
def process_df(df_):
encode_data(df_)
 for f_ in ['v2a1', 'v18q1', 'meaneduc', 'SQBmeaned']:
df_[f_] = df_[f_].fillna(0)
 df_['rez_esc'] = df_['rez_esc'].fillna(-1)
 return do_features(df_)
 train = process_df(train)
 test = process_df(test )<concatenate>
train = train[(train['trip_duration'] > 60)&(train['trip_duration'] < 3600)]
train['trip_duration'] = np.log(train['trip_duration'].values)
 train['hour'] = train['pickup_datetime'].apply(lambda x: int(x.split() [1][0:2]))
test['hour'] = test['pickup_datetime'].apply(lambda x: int(x.split() [1][0:2]))
train = train.loc[train['pickup_longitude']> -80]
train = train.loc[train['pickup_latitude']< 44]
train = train.loc[train['dropoff_longitude']> -90]
train = train.loc[train['dropoff_latitude']> 34]
def train_test_apply_func(train_, test_, func_):
test_['Target'] = 0
xx = pd.concat([train_, test_])
 xx_func = func_(xx)
 train_ = xx_func.iloc[:train_.shape[0], :]
test_ = xx_func.iloc[train_.shape[0]:, :].drop('Target', axis=1)
 del xx, xx_func
return train_, test_<feature_engineering>
def haversine(lat1, lon1, lat2, lon2):
R = 6372800
phi1, phi2 = math.radians(lat1), math.radians(lat2)
 dphi = math.radians(lat2 - lat1)
 dlambda = math.radians(lon2 - lon1)
 a = math.sin(dphi/2)**2 + \
math.cos(phi1)*math.cos(phi2)*math.sin(dlambda/2)**2
return 2*R*math.atan2(math.sqrt(a), math.sqrt(1 - a))
train['dist_long'] = train['pickup_longitude'] - train['dropoff_longitude']
test['dist_long'] = test['pickup_longitude'] - test['dropoff_longitude']
train['dist_lat'] = train['pickup_latitude'] - train['dropoff_latitude']
test['dist_lat'] = test['pickup_latitude'] - test['dropoff_latitude']
train['dist'] = np.sqrt(np.square(train['dist_long'])+ np.square(train['dist_lat']))
test['dist'] = np.sqrt(np.square(test['dist_long'])+ np.square(test['dist_lat']))
train['speed'] = 100000*train['dist'] / train['trip_duration']
train, test = train_test_apply_func(train, test, convert_OHE2LE )<categorify>
train.isnull().sum()
cols_2_ohe = ['eviv_LE', 'etecho_LE', 'epared_LE', 'elimbasu_LE',
'energcocinar_LE', 'sanitario_LE', 'manual_elec_LE',
'pared_LE']
cols_nums = ['age', 'meaneduc', 'dependency',
'hogar_nin', 'hogar_adul', 'hogar_mayor', 'hogar_total',
'bedrooms', 'overcrowding']
def convert_geo2aggs(df_):
tmp_df = pd.concat([df_[(['lugar_LE', 'idhogar']+cols_nums)],
pd.get_dummies(df_[cols_2_ohe],
columns=cols_2_ohe)],axis=1)
 geo_agg = tmp_df.groupby(['lugar_LE','idhogar'] ).mean().groupby('lugar_LE' ).mean().astype(np.float32)
 geo_agg.columns = pd.Index(['geo_' + e + '_MEAN' for e in geo_agg.columns.tolist() ])
 del tmp_df
return df_.join(geo_agg, how='left', on='lugar_LE')
 train, test = train_test_apply_func(train, test, convert_geo2aggs )<prepare_x_and_y>
train.isnull().sum()
X = train.query('parentesco1==1')
 y = X['Target'] - 1
X = X.drop(['Target'], axis=1 )<split>
y_train = train["trip_duration"]
X_train = train[["vendor_id","passenger_count","pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","dist","hour"]]
X_datatest = test[["vendor_id","passenger_count","pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","dist","hour"]]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=314, stratify=y )<compute_train_metric>
train.drop(['speed','dist','hour']+col_diff, axis=1, inplace=True )
def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) ,-1 ).argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', f1, True)
 fit_params={"early_stopping_rounds":300,
"eval_metric" : evaluate_macroF1_lgb,
"eval_set" : [(X_test,y_test)],
'eval_names': ['valid'],
'verbose': False,
'categorical_feature': 'auto'}
def learning_rate_power_0997(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.995, current_iter)
 return max(lr, min_learning_rate)
 fit_params['callbacks'] = [lgb.reset_parameter(learning_rate=learning_rate_power_0997)] 
<define_search_space>
X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size = 0.1, random_state=42 )
param_test ={'num_leaves': sp_randint(12, 20),
'min_child_samples': sp_randint(40, 100),
'subsample': sp_uniform(loc=0.75, scale=0.25),
'colsample_bytree': sp_uniform(loc=0.8, scale=0.15)
 }<choose_model_class>
params = {
'booster': 'gbtree',
'objective': 'reg:linear',
'learning_rate': 0.1,
'max_depth': 14,
'subsample': 0.8,
'colsample_bytree': 0.7,
'colsample_bylevel': 0.7,
'silent': 1
}
n_HP_points_to_test = 100
clf = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.05, objective='multiclass',
random_state=314, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced')
 gs = RandomizedSearchCV(
 estimator=clf, param_distributions=param_test,
n_iter=n_HP_points_to_test,
scoring='f1_macro',
cv=5,
refit=True,
random_state=314,
verbose=True )<train_model>
nrounds = 1200
dtrain = xgb.DMatrix(X_train, np.log(y_train+1))
gbm = xgb.train(params,
dtrain,
num_boost_round = nrounds )
_ = gs.fit(X_train, y_train, **fit_params )<find_best_params>
train_pred = np.exp(gbm.predict(xgb.DMatrix(X_datatest)))- 1
print("PERFORMANCE IMPROVES FROM TOP TO BOTTOM")
 print("Valid+-Std Train : Parameters")
 for i in np.argsort(gs.cv_results_['mean_test_score'])[-5:]:
print('{1:.3f}+-{3:.3f} {2:.3f} : {0}'.format(gs.cv_results_['params'][i],
gs.cv_results_['mean_test_score'][i],
gs.cv_results_['mean_train_score'][i],
gs.cv_results_['std_test_score'][i]))
opt_parameters = gs.best_params_<train_model>
my_submission = pd.DataFrame({"id": test.id, "trip_duration": np.exp(train_pred)})
 my_submission.head()
<load_from_csv><EOS> 
my_submission.to_csv('submission.csv', index=False)
 my_submission.head()