Kaggle competition: New York City Taxi Trip Duration
order_future = orders.loc[(( orders.eval_set == 'train')|(orders.eval_set == 'test')) , ['user_id', 'eval_set', 'order_id']]
order_future.head()<merge>
train_df = pd.read_csv('.. /input/nyc-taxi-trip-duration/train.csv', nrows = 1_000_000)
 train_df.dtypes
data = data.merge(order_future, on='user_id', how='left')
 data.head()<split>
y = np.log1p(train_df['trip_duration'])
data_train = data[data.eval_set == 'train']
data_train.head()<merge>
def calcul_distance(df):
pickedup =(df['pickup_latitude'], df['pickup_longitude'])
 dropoff =(df['dropoff_latitude'], df['dropoff_longitude'])
 return haversine(pickedup, dropoff )
data_train = data_train.merge(order_products_train[['product_id', 'order_id', 'reordered']], on=['product_id', 'order_id'], how='left')
 data_train.head()<normalization>
train_df['distance'] = train_df.apply(lambda x : calcul_distance(x), axis = 1)
data_train.reordered.fillna(0, inplace=True )<drop_column>
train_df['passenger_count'].value_counts()
train_df['vendor_id'].value_counts()
data_train = data_train.set_index(['user_id', 'product_id'])
 data_train.drop(['eval_set', 'order_id'], axis=1, inplace=True )<filter>
train_df['vendor_id'] = train_df['vendor_id'].astype('category' ).cat.codes
data_test = data[data.eval_set == 'test']
data_test.head()<drop_column>
data_test.drop(['eval_set', 'order_id'], axis=1, inplace=True )<drop_column>
train_df['speed'] = train_df['distance']/train_df['trip_duration']*3.6
data_test = data_test.set_index(['user_id', 'product_id'] )<drop_column>
train_new_1 = pd.read_csv(".. /input/new-york-city-taxi-with-osrm/fastest_routes_train_part_1.csv")
 train_new_2 = pd.read_csv(".. /input/new-york-city-taxi-with-osrm/fastest_routes_train_part_2.csv")
 train_test = pd.read_csv(".. /input/new-york-city-taxi-with-osrm/fastest_routes_test.csv")
 train_new = pd.concat([train_new_1, train_new_2], axis=0)
del [aisles, departments, order_products_prior, order_products_train, orders, order_future, data]<prepare_x_and_y>
train_all = train_df.merge(train_new, on='id', how='inner' )
data_train.reset_index(inplace=True)
 data_test.reset_index(inplace=True )<merge>
train_d = pd.to_datetime(train_all['pickup_datetime'])
 train_all['month'] = train_d.dt.month.astype('category' ).cat.codes
train_all['hour'] = train_d.dt.hour.astype('category' ).cat.codes
train_all['wday'] = train_d.dt.weekday.astype('category' ).cat.codes
data_train = data_train.merge(products[['product_id', 'aisle_id']], on='product_id', how='left')
 data_test = data_test.merge(products[['product_id', 'aisle_id']], on='product_id', how='left' )<merge>
SELECTED_COLUMNS = ['vendor_id', 'passenger_count', 'distance', 'pickup_latitude','pickup_longitude','dropoff_latitude', 'dropoff_longitude','hour','month','wday','total_distance','total_travel_time']
X = train_all[SELECTED_COLUMNS]
X.head(15 )
data_train = data_train.merge(products[['product_id', 'department_id']], on='product_id', how='left')
 data_test = data_test.merge(products[['product_id', 'department_id']], on='product_id', how='left' )<drop_column>
y = np.log1p(train_all['trip_duration'] )
data_test = data_test.set_index(['user_id', 'product_id'])
 data_train = data_train.set_index(['user_id', 'product_id'] )<predict_on_test>
rf = RandomForestRegressor(n_estimators = 100, random_state = 42 )
y_pred = xgb_clf.predict(data_test ).astype('int')
 y_pred =(xgb_clf.predict_proba(data_test)[:, 1] >= 0.21 ).astype('int')
 y_pred[0:10]<prepare_output>
rf.fit(X, y);
data_test['prediction'] = y_pred
data_test.head()<load_from_csv>
test_df = pd.read_csv('.. /input/nyc-taxi-trip-duration/test.csv')
 test_all = test_df.merge(train_test, on='id', how='inner')
 test_df.dtypes
test_df.head(10 )
orders = pd.read_csv(PATH + 'orders.csv')
 orders_test = orders.loc[orders.eval_set == 'test', ['user_id', 'order_id']]
orders_test.head()<merge>
test_df['vendor_id'] = test_df['vendor_id'].astype('category' ).cat.codes
final = final.merge(orders_test, on='user_id', how='left')
 final.head()<drop_column>
test_df['distance'] = test_df.apply(lambda x : calcul_distance(x), axis = 1 )
final = final.drop('user_id', axis=1 )<data_type_conversions>
test_df_da = pd.to_datetime(test_df['pickup_datetime'])
 test_df['month'] = test_df_da.dt.month.astype('category' ).cat.codes
test_df['hour'] = test_df_da.dt.hour.astype('category' ).cat.codes
test_df['wday'] = test_df_da.dt.weekday.astype('category' ).cat.codes
final['product_id'] = final.product_id.astype(int)
 del orders
del orders_test
gc.collect()
final.head()<feature_engineering>
test_all = test_df.merge(train_test, on='id', how='inner' )
d = dict()
for row in final.itertuples() :
if row.prediction== 1:
try:
d[row.order_id] += ' ' + str(row.product_id)
 except:
d[row.order_id] = str(row.product_id)
 for order in final.order_id:
if order not in d:
d[order] = 'None'
gc.collect()
d<prepare_output>
test_d_da = pd.to_datetime(test_all['pickup_datetime'])
 test_all['month'] = test_d_da.dt.month.astype('category' ).cat.codes
test_all['hour'] = test_d_da.dt.hour.astype('category' ).cat.codes
test_all['wday'] = test_d_da.dt.weekday.astype('category' ).cat.codes
sub = pd.DataFrame.from_dict(d, orient='index')
 sub.reset_index(inplace=True)
 sub.columns = ['order_id', 'products']
sub.head()<save_to_csv>
predictions = np.exp(rf.predict(X_test)) -np.ones(len(X_test))
X_test.shape
pred = pd.DataFrame(predictions, index=test_df['id'])
 pred.columns = ['trip_duration']
pred.to_csv("dat1.csv")
 pd.read_csv('dat1.csv' ).head()
sub.to_csv('sub.csv', index=False )<set_options>
scores = -cross_val_score(rf, X, y, cv=2, scoring = 'neg_mean_squared_error' )