Kaggle competition: New York City Taxi Trip Duration
<SOS> metric: RMSLE Kaggle data source: new-york-city-taxi-trip-duration<feature_engineering>
%matplotlib inline
span['Order_Range_D'] = span.total_orders - span.first_order_number + 1
span.head()<merge>
df = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
uxp_ratio = pd.merge(times, span, on=['user_id', 'product_id'], how='left')
 uxp_ratio.head()<feature_engineering>
df.duplicated().sum()
uxp_ratio['uxp_reorder_ratio'] = uxp_ratio.Times_Bought_N / uxp_ratio.Order_Range_D
uxp_ratio.head()<drop_column>
df.isna().sum()
uxp_ratio = uxp_ratio.drop(['Times_Bought_N', 'total_orders', 'first_order_number', 'Order_Range_D'], axis=1)
 uxp_ratio.head()<drop_column>
df = df[(df.trip_duration < 5900)]
del [times, first_order_no, span]<merge>
df = df[(df.passenger_count > 0)]
uxp = uxp.merge(uxp_ratio, on=['user_id', 'product_id'], how='left')
 del uxp_ratio
uxp.head()<groupby>
df = df[(df.pickup_longitude > -100)]
df = df[(df.pickup_latitude < 50)]
op['order_number_back'] = op.groupby('user_id')['order_number'].transform(max)- op.order_number +1
op.head(15 )<groupby>
df = pd.concat([df, pd.get_dummies(df['store_and_fwd_flag'])], axis=1)
 test = pd.concat([test, pd.get_dummies(test['store_and_fwd_flag'])], axis=1)
 df.drop(['store_and_fwd_flag'], axis=1, inplace=True)
 df = pd.concat([df, pd.get_dummies(df['vendor_id'])], axis=1)
 test = pd.concat([test, pd.get_dummies(test['vendor_id'])], axis=1)
 df.drop(['vendor_id'], axis=1, inplace=True )
last_five = op5.groupby(['user_id','product_id'])[['order_id']].count()
last_five.columns = ['times_last5']
last_five.head(10 )<merge>
df['pickup_datetime'] = pd.to_datetime(df.pickup_datetime)
 test['pickup_datetime'] = pd.to_datetime(test.pickup_datetime)
 df.drop(['dropoff_datetime'], axis=1, inplace=True)
 df['month'] = df.pickup_datetime.dt.month
df['week'] = df.pickup_datetime.dt.week
df['weekday'] = df.pickup_datetime.dt.weekday
df['hour'] = df.pickup_datetime.dt.hour
df['minute'] = df.pickup_datetime.dt.minute
df['minute_oftheday'] = df['hour'] * 60 + df['minute']
df.drop(['minute'], axis=1, inplace=True)
 test['month'] = test.pickup_datetime.dt.month
test['week'] = test.pickup_datetime.dt.week
test['weekday'] = test.pickup_datetime.dt.weekday
test['hour'] = test.pickup_datetime.dt.hour
test['minute'] = test.pickup_datetime.dt.minute
test['minute_oftheday'] = test['hour'] * 60 + test['minute']
test.drop(['minute'], axis=1, inplace=True)
 df.drop(['pickup_datetime'], axis=1, inplace=True)
 df.info()
uxp = uxp.merge(last_five, on=['user_id', 'product_id'], how='left')
 del last_five
uxp.head()<correct_missing_values>
def ft_haversine_distance(lat1, lng1, lat2, lng2):
lat1, lng1, lat2, lng2 = map(np.radians,(lat1, lng1, lat2, lng2))
AVG_EARTH_RADIUS = 6371
lat = lat2 - lat1
lng = lng2 - lng1
d = np.sin(lat * 0.5)** 2 + np.cos(lat1)* np.cos(lat2)* np.sin(lng * 0.5)** 2
h = 2 * AVG_EARTH_RADIUS * np.arcsin(np.sqrt(d))
return h
df['distance'] = ft_haversine_distance(df['pickup_latitude'].values,
df['pickup_longitude'].values,
df['dropoff_latitude'].values,
df['dropoff_longitude'].values)
 test['distance'] = ft_haversine_distance(test['pickup_latitude'].values,
test['pickup_longitude'].values,
test['dropoff_latitude'].values,
test['dropoff_longitude'].values )
uxp = uxp.fillna(0)
 uxp.head()<merge>
def ft_degree(lat1, lng1, lat2, lng2):
AVG_EARTH_RADIUS = 6371
lng_delta_rad = np.radians(lng2 - lng1)
 lat1, lng1, lat2, lng2 = map(np.radians,(lat1, lng1, lat2, lng2))
y = np.sin(lng_delta_rad)* np.cos(lat2)
 x = np.cos(lat1)* np.sin(lat2)- np.sin(lat1)* np.cos(lat2)* np.cos(lng_delta_rad)
 return np.degrees(np.arctan2(y, x))
df['direction'] = ft_degree(df['pickup_latitude'].values,
df['pickup_longitude'].values,
df['dropoff_latitude'].values,
df['dropoff_longitude'].values)
 test['direction'] = ft_degree(test['pickup_latitude'].values,
test['pickup_longitude'].values,
test['dropoff_latitude'].values,
test['dropoff_longitude'].values )
data = uxp.merge(user, on='user_id', how='left')
 data.head()<merge>
df = df[(df.distance < 200)]
data = data.merge(prd, on='product_id', how='left')
 data.head()<set_options>
df['speed'] = df.distance / df.trip_duration
del op, user, prd, uxp
gc.collect()<filter>
df = df[(df.speed < 30)]
df.drop(['speed'], axis=1, inplace=True )
orders_future = orders[(( orders.eval_set=='train')|(orders.eval_set=='test')) ]
orders_future = orders_future[ ['user_id', 'eval_set', 'order_id'] ]
orders_future.head(10)
<merge>
y = df["trip_duration"]
df.drop(["trip_duration"], axis=1, inplace=True)
 df.drop(['id'], axis=1, inplace=True)
 X = df
X.shape, y.shape
data = data.merge(orders_future, on='user_id', how='left')
 data.head(10 )<filter>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
 X_train.shape, y_train.shape, X_test.shape, y_test.shape
data_train = data[data.eval_set=='train']
data_train.head()<merge>
from sklearn.metrics import mean_squared_error as MSE
data_train = data_train.merge(order_products_train[['product_id','order_id', 'reordered']], on=['product_id','order_id'], how='left')
 data_train.head(15 )<feature_engineering>
data_train['reordered'] = data_train['reordered'].fillna(0)
 data_train.head(15 )<drop_column>
data_train = data_train.set_index(['user_id', 'product_id'])
 data_train.head(15 )<drop_column>
import lightgbm as lgb
data_train = data_train.drop(['eval_set', 'order_id'], axis=1)
 data_train.head(15 )<filter>
data_test = data[data.eval_set=='test']
data_test.head()<drop_column>
data_test = data_test.set_index(['user_id', 'product_id'])
 data_test.head()<drop_column>
lgb_params = {
'learning_rate': 0.1,
'max_depth': 25,
'num_leaves': 1000,
'objective': 'regression',
'feature_fraction': 0.9,
'bagging_fraction': 0.5,
'max_bin': 1000 }
data_test = data_test.drop(['eval_set','order_id'], axis=1)
 data_test.head()<prepare_x_and_y>
X_train, y_train = data_train.drop('reordered', axis=1), data_train.reordered
parameters = {'eval_metric':'logloss',
'max_depth':'5',
'colsample_bytreeι':'0.4',
'subsample':'0.8'
}
xgbc = xgb.XGBClassifier(objective='binary:logistic', parameters=parameters, num_boost_round=10)
 model = xgbc.fit(X_train, y_train)
 xgb.plot_importance(model )<find_best_params>
model.get_xgb_params()<prepare_x_and_y>
%%time
lgb_df = lgb.Dataset(X, y)
 lgb_model = lgb.train(lgb_params, lgb_df, num_boost_round=1500 )
print("The best parameters are: /n", gridsearch.best_params_)
 model = gridsearch.best_estimator_
del [X_train, y_train]<find_best_params>
test_columns = X.columns
predictions = lgb_model.predict(test[test_columns] )
model.get_params()<predict_on_test>
submission = pd.DataFrame({'id': test.id, 'trip_duration': np.exp(predictions)})
 submission.head()
<predict_on_test><EOS> 
submission.to_csv("sub.csv", index=False )