Kaggle competition: New York City Taxi Trip Duration
<SOS> metric: RMSLE Kaggle data source: new-york-city-taxi-trip-duration<filter>
import numpy as np
import pandas as pd
import datetime as dt
import seaborn as sns
import matplotlib.pyplot as plt
from geopy import distance
import os
from sklearn.model_selection import cross_val_score
from sklearn.ensemble.forest import RandomForestClassifier
from sklearn.ensemble import RandomForestRegressor
from sklearn.datasets import make_regression
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
orders_test = orders.loc[orders.eval_set=='test',("user_id", "order_id")]
orders_test.head()<merge>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
final = final.merge(orders_test, on='user_id', how='left')
 final.head()<drop_column>
train.isna().sum()
final = final.drop('user_id', axis=1 )<data_type_conversions>
test.isna().sum()
final['product_id'] = final.product_id.astype(int)
 del orders
del orders_test
gc.collect()
final.head()<feature_engineering>
train['pickup_datetime']= pd.to_datetime(train.pickup_datetime, format='%Y-%m-%d %H:%M:%S')
 train['day_of_the_date']=train.pickup_datetime.dt.dayofweek
train['month'] = train.pickup_datetime.dt.month
train['day'] = train.pickup_datetime.dt.day
train['hour'] = train.pickup_datetime.dt.hour
train['minute'] = train.pickup_datetime.dt.minute
train.head(5 )
d = dict()
for row in final.itertuples() :
if row.prediction== 1:
try:
d[row.order_id] += ' ' + str(row.product_id)
 except:
d[row.order_id] = str(row.product_id)
 for order in final.order_id:
if order not in d:
d[order] = 'None'
gc.collect()
d<prepare_output>
test['pickup_datetime']= pd.to_datetime(test.pickup_datetime, format='%Y-%m-%d %H:%M:%S')
 test['day_of_the_date']=test.pickup_datetime.dt.dayofweek
test['month'] = test.pickup_datetime.dt.month
test['day'] = test.pickup_datetime.dt.day
test['hour'] = test.pickup_datetime.dt.hour
test['minute'] = test.pickup_datetime.dt.minute
test.head(5 )
sub = pd.DataFrame.from_dict(d, orient='index')
 sub.reset_index(inplace=True)
 sub.columns = ['order_id', 'products']
sub.head()<save_to_csv>
def distancer(row):
coords_1 =(row['pickup_latitude'], row['pickup_longitude'])
 coords_2 =(row['dropoff_latitude'], row['dropoff_longitude'])
 return distance.distance(coords_1, coords_2 ).km
train['distance'] = train.apply(distancer, axis=1)
 test['distance'] = test.apply(distancer, axis=1 )
sub.to_csv('sub.csv', index=False )<set_options>
input_columns=['day_of_the_date', 'month', 'day', 'hour','distance', 'pickup_longitude', 'pickup_latitude', 'dropoff_longitude', 'dropoff_latitude']
y=train['trip_duration_log']
X=train[input_columns]
X_test=test[input_columns]
gc.enable()<load_from_csv>
X_train, X_valid, y_train, y_valid = train_test_split(X,y, test_size=0.2, random_state=42 )
orders = pd.read_csv('.. /input/orders.csv')
 order_products_train = pd.read_csv('.. /input/order_products__train.csv')
 order_products_prior = pd.read_csv('.. /input/order_products__prior.csv')
 products = pd.read_csv('.. /input/products.csv')
 aisles = pd.read_csv('.. /input/aisles.csv')
 departments = pd.read_csv('.. /input/departments.csv' )<data_type_conversions>
rfr = RandomForestRegressor(n_estimators=100,min_samples_leaf=3, min_samples_split=15, n_jobs=-1, max_features="auto")
 rfr.fit(X_train, y_train )
aisles['aisle'] = aisles['aisle'].astype('category')
 departments['department'] = departments['department'].astype('category')
 orders['eval_set'] = orders['eval_set'].astype('category')
 products['product_name'] = products['product_name'].astype('category' )<merge>
cv_scores = cross_val_score(rfr, X_train, y_train, cv=5)
 for i in range(len(cv_scores)) :
cv_scores[i]=np.sqrt(abs(cv_scores[i]))
print(np.mean(cv_scores))
op = orders.merge(order_products_prior, on='order_id', how='inner')
 op.head()<merge>
train_pred=rfr.predict(X_test)
 train_pred
pbp = op.groupby(['user_id','product_id'])['product_id'].count().to_frame('times_p_order')
 pbp = pbp.reset_index()
pbp_new = pbp.groupby('user_id')['times_p_order'].sum().to_frame('total_products')
 pbp= pd.merge(pbp,pbp_new,on=["user_id"],how="left")
 pbp["%_of_bought_products_in_total"] = pbp.times_p_order/pbp.total_products
del pbp_new<merge>
my_submission = pd.DataFrame({'id':test.id, 'trip_duration':np.exp(train_pred)} )
grouped=op.groupby("product_id")["reordered"].count().to_frame("number_of_reorders" ).reset_index()
pbp = pd.merge(pbp,grouped,on="product_id",how="left")
 del grouped 
<merge>
my_submission.to_csv('sub.csv',index=False )