Kaggle competition: New York City Taxi Trip Duration
<SOS> metric: RMSLE Kaggle data source: new-york-city-taxi-trip-duration<feature_engineering>
%matplotlib inline
span['Order_Range_D'] = span.total_orders - span.first_order_number + 1
span.head()<merge>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 sample = pd.read_csv('.. /input/sample_submission.csv')
uxp_ratio = pd.merge(times, span, on=['user_id', 'product_id'], how='left')
 uxp_ratio.head()<feature_engineering>
train.isna().sum()
uxp_ratio['uxp_order_ratio'] = uxp_ratio.Times_Bought_N / uxp_ratio.Order_Range_D
uxp_ratio.head()<drop_column>
train['trip_duration'] = np.log(train['trip_duration'].values )
uxp_ratio = uxp_ratio.drop(['Times_Bought_N', 'total_orders', 'first_order_number', 'Order_Range_D'], axis=1)
 uxp_ratio.head()<drop_column>
train['passenger_count'].value_counts()
del [times, first_order_no, span] 
<merge>
def haversine(lat1, lon1, lat2, lon2):
R = 6372800
phi1, phi2 = math.radians(lat1), math.radians(lat2)
 dphi = math.radians(lat2 - lat1)
 dlambda = math.radians(lon2 - lon1)
 a = math.sin(dphi/2)**2 + \
math.cos(phi1)*math.cos(phi2)*math.sin(dlambda/2)**2
return 2*R*math.atan2(math.sqrt(a), math.sqrt(1 - a))
train['dist_long'] = train['pickup_longitude'] - train['dropoff_longitude']
test['dist_long'] = test['pickup_longitude'] - test['dropoff_longitude']
train['dist_lat'] = train['pickup_latitude'] - train['dropoff_latitude']
test['dist_lat'] = test['pickup_latitude'] - test['dropoff_latitude']
train['dist'] = np.sqrt(np.square(train['dist_long'])+ np.square(train['dist_lat']))
test['dist'] = np.sqrt(np.square(test['dist_long'])+ np.square(test['dist_lat']))
uxp = uxp.merge(uxp_ratio, on=['user_id', 'product_id'], how='left')
 del uxp_ratio
uxp.head()<merge>
train['pickup_datetime'] = pd.to_datetime(train['pickup_datetime'])
 train['dropoff_datetime'] = pd.to_datetime(train['dropoff_datetime'])
 test['pickup_datetime'] = pd.to_datetime(test['pickup_datetime'])
 train['hour'] = train.pickup_datetime.dt.hour
train['day'] = train.pickup_datetime.dt.dayofweek
train['month'] = train.pickup_datetime.dt.month
test['hour'] = test.pickup_datetime.dt.hour
test['day'] = test.pickup_datetime.dt.dayofweek
test['month'] = test.pickup_datetime.dt.month
data = uxp.merge(user, on='user_id', how='left')
 data.head()<merge>
train.isnull().sum()
data = data.merge(prd, on='product_id', how='left')
 data.head()<set_options>
train.isnull().sum()
del op, user, prd, uxp
gc.collect()<filter>
y_train = train["trip_duration"]
X_train = train[["vendor_id","passenger_count","pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","month","hour","day","dist"]]
X_datatest = test[["vendor_id","passenger_count","pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","month","hour","day","dist"]]
orders_last = orders[(orders.eval_set=='train')|(orders.eval_set=='test')]<merge>
data = data.merge(orders_last, on='user_id', how='left')
 data.head()<merge>
X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size = 0.1, random_state=42 )
data_train = data[data.eval_set=='train']
data_train = data_train.merge(order_products_train, on=['product_id', 'order_id'], how='left')
 data_train = data_train.drop(['order_id','eval_set', 'add_to_cart_order'], axis=1)
 data_train = data_train.fillna(0)
 data_train.head()<drop_column>
rfr = RandomForestRegressor(n_estimators=100,min_samples_leaf=10, min_samples_split=15, max_depth=80,verbose=0,max_features="auto",bootstrap=True,n_jobs=-1)
 rfr.fit(X_train, y_train )
data_test = data[data.eval_set=='test']
data_test = data_test.drop(['eval_set', 'order_id'], axis=1)
 data_test = data_test.fillna(0)
 data_test.head()<set_options>
cv_scores = cross_val_score(rfr, X_train, y_train, cv=5, scoring= 'neg_mean_squared_log_error' )
del data
del orders_last
gc.collect()<drop_column>
for i in range(len(cv_scores)) :
cv_scores[i] = np.sqrt(abs(cv_scores[i]))
cv_scores
data_train = data_train.set_index(['user_id', 'product_id'])
 data_test = data_test.set_index(['user_id', 'product_id'] )<split>
train_pred = rfr.predict(X_datatest)
 train_pred[:5]
data_train.loc[:, 'reordered'] = data_train.reordered.fillna(0)
 X_train, X_val, y_train, y_val = train_test_split(data_train.drop('reordered', axis=1), data_train.reordered,
test_size=0.2, random_state=42)
 del data_train
gc.collect()
d_train = xgboost.DMatrix(X_train, y_train)
 xgb_params = {
"objective" : "reg:logistic"
,"eval_metric" : "logloss"
,"eta" : 0.1
,"max_depth" : 6
,"min_child_weight" :10
,"gamma" :0.70
,"subsample" :0.76
,"colsample_bytree" :0.95
,"alpha" :2e-05
,"lambda" :10
}
watchlist= [(d_train, "train")]
bst = xgboost.train(params=xgb_params, dtrain=d_train, num_boost_round=80, evals=watchlist, verbose_eval=10)
 xgboost.plot_importance(bst )<drop_column>
my_submission = pd.DataFrame({"id": test.id, "trip_duration": np.exp(train_pred)})
 print(my_submission )
<prepare_x_and_y><EOS> 
my_submission.to_csv('submission.csv', index=False)