Kaggle competition: New York City Taxi Trip Duration
gc.collect()<set_options>
%matplotlib inline
gc.enable()<load_from_csv>
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )
orders = pd.read_csv('.. /input/orders.csv')
 order_products_train = pd.read_csv('.. /input/order_products__train.csv')
 order_products_prior = pd.read_csv('.. /input/order_products__prior.csv')
 products = pd.read_csv('.. /input/products.csv')
 aisles = pd.read_csv('.. /input/aisles.csv')
 departments = pd.read_csv('.. /input/departments.csv' )<data_type_conversions>
df_train.isna().sum()
aisles['aisle'] = aisles['aisle'].astype('category')
 departments['department'] = departments['department'].astype('category')
 orders['eval_set'] = orders['eval_set'].astype('category')
 products['product_name'] = products['product_name'].astype('category' )<merge>
df_train.duplicated().sum()
op = orders.merge(order_products_prior, on='order_id', how='inner')
 op.head()<create_dataframe>
df_train = df_train[(df_train['trip_duration'] > 60)&(df_train['trip_duration'] < 3600)]
df_train['trip_duration'] = np.log(df_train['trip_duration'].values )
user = op.groupby('user_id')['order_number'].max().to_frame('u_total_orders')
 user.head() 
<prepare_output>
from datetime import datetime
dow=op.groupby('user_id')['order_dow'].mean().to_frame('average_dow')
 dow=dow.reset_index()
dow.head()<merge>
df_train['pickup_datetime'] = pd.to_datetime(df_train['pickup_datetime'])
 df_train['dropoff_datetime'] = pd.to_datetime(df_train['dropoff_datetime'])
 df_test['pickup_datetime'] = pd.to_datetime(df_test['pickup_datetime'] )
user = user.merge(u_reorder, on='user_id', how='left')
 del u_reorder
gc.collect()
user.head()<merge>
df_train['minute'] = df_train.pickup_datetime.dt.minute
df_train['hour'] = df_train.pickup_datetime.dt.hour
df_train['day'] = df_train.pickup_datetime.dt.dayofweek
df_train['month'] = df_train.pickup_datetime.dt.month
df_test['minute'] = df_test.pickup_datetime.dt.minute
df_test['hour'] = df_test.pickup_datetime.dt.hour
df_test['day'] = df_test.pickup_datetime.dt.dayofweek
df_test['month'] = df_test.pickup_datetime.dt.month
user = user.merge(dow, on='user_id', how='left')
 del dow
gc.collect()
user.head()<prepare_output>
df_train['d_longitude'] = df_train['pickup_longitude'] - df_train['dropoff_longitude']
df_train['d_latitude'] = df_train['pickup_latitude'] - df_train['dropoff_latitude']
df_test['d_longitude'] = df_test['pickup_longitude'] - df_test['dropoff_longitude']
df_test['d_latitude'] = df_test['pickup_latitude'] - df_test['dropoff_latitude']
df_train['distance'] = np.sqrt(np.square(df_train['d_longitude'])+ np.square(df_train['d_latitude']))
df_test['distance'] = np.sqrt(np.square(df_test['d_longitude'])+ np.square(df_test['d_latitude']))
prd = op.groupby('product_id')['order_id'].count().to_frame('p_total_purchases')
 prd.head()<groupby>
FEATURES = ["vendor_id","passenger_count","pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","distance","month","hour","day"]
TARGET = "trip_duration"
X_train = df_train[FEATURES]
y_train = df_train[TARGET]
p_reorder = op.groupby('product_id' ).filter(lambda x: x.shape[0] >40)
 p_reorder.head()<merge>
lgb_train = lgb.Dataset(X_train,y_train )
prd = prd.merge(p_reorder, on='product_id', how='left')
 del p_reorder
gc.collect()
prd.head()<merge>
lgb_params = {
'learning_rate': 0.1,
'max_depth': 25,
'num_leaves': 1000,
'objective': 'regression',
'feature_fraction': 0.9,
'bagging_fraction': 0.5,
'max_bin': 1000}
prd=prd.merge(Addtocart, on='product_id',how='left')
 del Addtocart
gc.collect()
prd.head()<feature_engineering>
model_lgb = lgb.train(lgb_params,lgb_train,num_boost_round=500 )
prd['p_reorder_ratio'] = prd['p_reorder_ratio'].fillna(value=0)
 prd.head()<prepare_output>
cv_score = lgb.cv(
 lgb_params,
lgb_train,
num_boost_round=100,
nfold=3,
metrics='mae',
early_stopping_rounds=10,
stratified=False
)
uxp = op.groupby(['user_id', 'product_id'])['order_id'].count().to_frame('uxp_total_bought')
 uxp.head()<groupby>
print('Best CV score:', cv_score['l1-mean'][-1] )
times = op.groupby(['user_id', 'product_id'])[['order_id']].count()
times.columns = ['Times_Bought_N']
times.head()<prepare_output>
X_prediction = df_test[FEATURES]
prediction = np.exp(model_lgb.predict(X_prediction))
prediction
<merge><EOS> 
submission = pd.DataFrame({'id': df_test.id, 'trip_duration': prediction})
 submission.to_csv('submission.csv', index=False )