Kaggle competition: New York City Taxi Trip Duration
final = final.merge(orders_test, on='user_id', how='left')
 final.head()<drop_column>
%matplotlib inline
final = final.drop('user_id', axis=1)
<data_type_conversions>
test = pd.read_csv('.. /input/test.csv')
 train = pd.read_csv('.. /input/train.csv' )
final['product_id'] = final.product_id.astype(int)
 del orders
del orders_test
gc.collect()
final.head()<feature_engineering>
train.isna().sum()
d = dict()
for row in final.itertuples() :
if row.prediction== 1:
try:
d[row.order_id] += ' ' + str(row.product_id)
 except:
d[row.order_id] = str(row.product_id)
 for order in final.order_id:
if order not in d:
d[order] = 'None'
gc.collect()
d<prepare_output>
train.duplicated().sum()
sub = pd.DataFrame.from_dict(d, orient='index')
 sub.reset_index(inplace=True)
 sub.columns = ['order_id', 'products']
sub.head()<save_to_csv>
train.isnull().sum()
sub.to_csv('sub.csv', index=False )<set_options>
train.isnull().sum()
gc.enable()<load_from_csv>
train = train[(train['log_trip_duration'] > 3.5)]
train = train[(train['log_trip_duration'] < 9)]
orders = pd.read_csv('.. /input/orders.csv')
 order_products_train = pd.read_csv('.. /input/order_products__train.csv')
 order_products_prior = pd.read_csv('.. /input/order_products__prior.csv')
 products = pd.read_csv('.. /input/products.csv')
 aisles = pd.read_csv('.. /input/aisles.csv')
 departments = pd.read_csv('.. /input/departments.csv' )<data_type_conversions>
train = train.loc[train['pickup_longitude']> -74.02]
train = train.loc[train['pickup_latitude']< 40.77]
aisles['aisle'] = aisles['aisle'].astype('category')
 departments['department'] = departments['department'].astype('category')
 orders['eval_set'] = orders['eval_set'].astype('category')
 products['product_name'] = products['product_name'].astype('category' )<merge>
train = train.loc[train['dropoff_longitude']> -74.02]
train = train.loc[train['dropoff_latitude']< 40.77]
op = orders.merge(order_products_prior, on='order_id', how='inner')
 op.head()<prepare_output>
train = train.loc[train['passenger_count']<6]
train = train[(train.passenger_count > 0)]
user = op.groupby('user_id')['order_number'].max().to_frame('u_total_orders')
 user = user.reset_index()
user.head() 
<merge>
train['dist'] = np.sqrt(np.square(train['pickup_longitude'] - train['dropoff_longitude'])+ np.square(train['pickup_latitude'] - train['dropoff_latitude']))
test['dist'] = np.sqrt(np.square(test['pickup_longitude'] - test['dropoff_longitude'])+ np.square(test['pickup_latitude'] - test['dropoff_latitude']))
train['pickup_datetime'] = pd.to_datetime(train['pickup_datetime'])
 test['pickup_datetime'] = pd.to_datetime(test['pickup_datetime'])
 train['month'] = train['pickup_datetime'].dt.month
test['month'] = test['pickup_datetime'].dt.month
train['day'] = train['pickup_datetime'].dt.dayofweek
test['day'] = test['pickup_datetime'].dt.dayofweek
train['hour'] = train['pickup_datetime'].dt.hour
test['hour'] = test['pickup_datetime'].dt.hour
user = user.merge(u_reorder, on='user_id', how='left')
 del u_reorder
gc.collect()
user.head()<prepare_output>
train = train.loc[train['hour']>=8]
prd = op.groupby('product_id')['order_id'].count().to_frame('p_total_purchases')
 prd = prd.reset_index()
prd.head()<groupby>
y = train["log_trip_duration"]
X_train = train[["vendor_id","passenger_count","pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","month","hour","day","dist"]]
X_test = test[["vendor_id","passenger_count","pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","month","hour","day","dist"]]
p_reorder = op.groupby('product_id' ).filter(lambda x: x.shape[0] >40)
 p_reorder.head()<create_dataframe>
xtrain, xvalid, ytrain, yvalid = train_test_split(X_train,y, test_size=0.2, random_state=42)
 xtrain.shape, xvalid.shape, xtrain.shape, yvalid.shape
p_reorder = p_reorder.groupby('product_id')['reordered'].mean().to_frame('p_reorder_ratio')
 p_reorder = p_reorder.reset_index()
p_reorder.head()<merge>
dtrain = lgb.Dataset(X_train, y)
 lgb_params = {
'learning_rate': 0.1,
'max_depth': 25,
'feature_fraction': 0.9,
'bagging_fraction': 0.5,
'num_leaves': 1000,
'objective': 'regression',
'max_bin': 1000}
prd = prd.merge(p_reorder, on='product_id', how='left')
 del p_reorder
gc.collect()
prd.head()<feature_engineering>
resultsCV = lgb.cv(lgb_params,dtrain,num_boost_round=100,nfold=3,metrics='mae',early_stopping_rounds=10,stratified=False)
 print('best score :', resultsCV['l1-mean'][-1] )
prd['p_reorder_ratio'] = prd['p_reorder_ratio'].fillna(value=0)
 prd.head()<create_dataframe>
model_lgb = lgb.train(lgb_params,
dtrain,
num_boost_round=1200 )
uxp = op.groupby(['user_id', 'product_id'])['order_id'].count().to_frame('uxp_total_bought')
 uxp = uxp.reset_index()
uxp.head()<feature_engineering>
pred_test = np.exp(model_lgb.predict(X_test))
pred_test
op['order_number_back'] = op.groupby('user_id')['order_number'].transform(max)- op.order_number +1
op5 = op[op.order_number_back <= 5]
last_five = op5.groupby(['user_id','product_id'])[['order_id']].count()
last_five.columns = ['times_last5']
last_five['times_last5_ratio'] = last_five.times_last5 / 5
uxp = uxp.merge(last_five , on=['user_id', 'product_id'], how='left')
 del [last_five, op5]
gc.collect()
uxp.head()<data_type_conversions>
submit = pd.read_csv('.. /input/sample_submission.csv' )
uxp['times_last5'] = uxp['times_last5'].fillna(value=0)
 uxp['times_last5_ratio'] = uxp['times_last5_ratio'].fillna(value=0)
 uxp.head()<groupby>
submit['trip_duration'] = pred_test
submit.head(20 )
times = op.groupby(['user_id', 'product_id'])[['order_id']].count()
times.columns = ['Times_Bought_N']
times.head()<prepare_output>
submit_file = pd.DataFrame({"id": test.id, "trip_duration": pred_test} )
<merge><EOS> 
submit_file.to_csv('submission.csv', index=False )