Kaggle competition: New York City Taxi Trip Duration
mean_time_order=op.groupby(['user_id','product_id'])['order_hour_of_day'].mean().to_frame('mean_time_of_order' ).round(decimals=0)
 pbp=pd.merge(pbp,mean_time_order,on=['user_id','product_id'],how="left")
 del mean_time_order<create_dataframe>
TRAINPATH = os.path.join(".. ", "input", "train.csv")
 TESTPATH = os.path.join(".. ", "input", "test.csv" )
mean_prod = op.groupby(['user_id','order_number'])['add_to_cart_order'].max().to_frame("product_number_per_order")
 mean_prod = mean_prod.groupby("user_id")["product_number_per_order"].mean().to_frame("mean_product_cart" ).reset_index()
pbp=pd.merge(pbp,mean_prod,on="user_id",how="left")
 del mean_prod 
<merge>
df = pd.read_csv(TRAINPATH, index_col=0)
 df.head()
mean_cart_ord = op.groupby(['user_id', 'product_id'])['add_to_cart_order'].mean().to_frame("mean_preference" ).reset_index()
pbp=pd.merge(pbp,mean_cart_ord,on=['user_id', 'product_id'],how="left")
 pbp.head()
del mean_cart_ord<drop_column>
df_test = pd.read_csv(TESTPATH)
 df_test.head()
pbp = pbp.drop(["times_p_order","total_products"],axis=1)
 pbp.head()<create_dataframe>
df.duplicated().sum()
user = op.groupby('user_id')['order_number'].max().to_frame('u_total_orders')
 user.head() 
<merge>
df = df.drop_duplicates()
df.duplicated().sum()
user = user.merge(u_reorder, on='user_id', how='left')
 del u_reorder
gc.collect()
user.head()<prepare_output>
df.isna().sum()
prd = op.groupby('product_id')['order_id'].count().to_frame('p_total_purchases')
 prd.head()<groupby>
CAT_VARS = ['store_and_fwd_flag']
p_reorder = op.groupby('product_id' ).filter(lambda x: x.shape[0] >40)
 p_reorder.head()<merge>
for col in CAT_VARS:
df[col] = df[col].astype('category' ).cat.codes
df.head()
prd = prd.merge(p_reorder, on='product_id', how='left')
 del p_reorder
gc.collect()
prd.head()<feature_engineering>
for col in CAT_VARS:
df_test[col] = df_test[col].astype('category' ).cat.codes
df_test.head()
prd['p_reorder_ratio'] = prd['p_reorder_ratio'].fillna(value=0)
 prd.head()<prepare_output>
from sklearn.decomposition import PCA
uxp = op.groupby(['user_id', 'product_id'])['order_id'].count().to_frame('uxp_total_bought')
 uxp.head()<groupby>
coords = np.vstack(( df[['pickup_latitude', 'pickup_longitude']].values,
df[['dropoff_latitude', 'dropoff_longitude']].values,
df_test[['pickup_latitude', 'pickup_longitude']].values,
df_test[['dropoff_latitude', 'dropoff_longitude']].values))
pca = PCA().fit(coords)
 df['pickup_pca0'] = pca.transform(df[['pickup_latitude', 'pickup_longitude']])[:, 0]
df['pickup_pca1'] = pca.transform(df[['pickup_latitude', 'pickup_longitude']])[:, 1]
df['dropoff_pca0'] = pca.transform(df[['dropoff_latitude', 'dropoff_longitude']])[:, 0]
df['dropoff_pca1'] = pca.transform(df[['dropoff_latitude', 'dropoff_longitude']])[:, 1]
df_test['pickup_pca0'] = pca.transform(df_test[['pickup_latitude', 'pickup_longitude']])[:, 0]
df_test['pickup_pca1'] = pca.transform(df_test[['pickup_latitude', 'pickup_longitude']])[:, 1]
df_test['dropoff_pca0'] = pca.transform(df_test[['dropoff_latitude', 'dropoff_longitude']])[:, 0]
df_test['dropoff_pca1'] = pca.transform(df_test[['dropoff_latitude', 'dropoff_longitude']])[:, 1]
times = op.groupby(['user_id', 'product_id'])[['order_id']].count()
times.columns = ['Times_Bought_N']
times.head()<prepare_output>
df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'])
 df['dropoff_datetime'] = pd.to_datetime(df['dropoff_datetime'])
 df_test['pickup_datetime'] = pd.to_datetime(df_test['pickup_datetime'] )
total_orders = op.groupby('user_id')['order_number'].max().to_frame('total_orders')
 total_orders.head()<merge>
df['hour'] = df.pickup_datetime.dt.hour
df['day'] = df.pickup_datetime.dt.dayofweek
df['month'] = df.pickup_datetime.dt.month
df_test['hour'] = df_test.pickup_datetime.dt.hour
df_test['day'] = df_test.pickup_datetime.dt.dayofweek
df_test['month'] = df_test.pickup_datetime.dt.month
span = pd.merge(total_orders, first_order_no, on='user_id', how='right')
 span.head()<feature_engineering>
df['distance2'] = np.sqrt(( df['pickup_pca0']-df['dropoff_pca0'])**2
+(df['pickup_pca1']-df['dropoff_pca1'])**2)
 df_test['distance2'] = np.sqrt(( df_test['pickup_pca0']-df_test['dropoff_pca0'])**2
+(df_test['pickup_pca1']-df_test['dropoff_pca1'])**2 )
span['Order_Range_D'] = span.total_orders - span.first_order_number + 1
span.head()<merge>
df['log_trip_duration'] = np.log(df['trip_duration'])
uxp_ratio = pd.merge(times, span, on=['user_id', 'product_id'], how='left')
 uxp_ratio.head()<feature_engineering>
NUM_VARS = ['pickup_pca0', 'pickup_pca1', 'dropoff_pca0', 'dropoff_pca1', 'month', 'hour', 'day', 'distance2']
TARGET = 'log_trip_duration'
uxp_ratio['uxp_reorder_ratio'] = uxp_ratio.Times_Bought_N / uxp_ratio.Order_Range_D
uxp_ratio.head()<drop_column>
num_features = NUM_VARS
uxp_ratio = uxp_ratio.drop(['Times_Bought_N', 'total_orders', 'first_order_number', 'Order_Range_D'], axis=1)
 uxp_ratio.head()<drop_column>
X_train = df.loc[:, num_features]
y_train = df[TARGET]
X_test = df_test.loc[:, num_features]
X_train.shape, y_train.shape, X_test.shape
del [times, first_order_no, span]<merge>
from sklearn.ensemble import RandomForestRegressor
uxp = uxp.merge(uxp_ratio, on=['user_id', 'product_id'], how='left')
 del uxp_ratio
uxp.head()<merge>
training = RandomForestRegressor(n_estimators=19, min_samples_leaf=10, min_samples_split=15, max_features='auto', max_depth=80, bootstrap=True)
 training.fit(X_train, y_train )
data = uxp.merge(user, on='user_id', how='left')
 data.head()<merge>
from sklearn.model_selection import cross_val_score
data = data.merge(pbp, on=["user_id","product_id"], how="left")
 data.head()<merge>
cv_scores = cross_val_score(training, X_train, y_train, cv=5, scoring='neg_mean_squared_log_error' )
data = data.merge(prd, on='product_id', how='left')
 data.head()<set_options>
for i in range(len(cv_scores)) :
cv_scores[i] = np.sqrt(abs(cv_scores[i]))
cv_scores
del op, user, prd, uxp
gc.collect()<filter>
y_test_pred = training.predict(X_test)
 y_test_pred[:5]
orders_future = orders[(( orders.eval_set=='train')|(orders.eval_set=='test')) ]
orders_future = orders_future[ ['user_id', 'eval_set', 'order_id'] ]
orders_future.head(10)
<merge>
predection = pd.DataFrame({'id': df_test.id, 'trip_duration': np.exp(y_test_pred)})
 predection.to_csv('submission.csv', index=False )