Kaggle competition: New York City Taxi Trip Duration
<SOS> metric: RMSLE Kaggle data source: new-york-city-taxi-trip-duration<load_from_csv>
plt.rcParams['figure.figsize'] = [16, 10]
%matplotlib inline
cd_train = pd.read_csv('coupon_detail_train.csv')
 cl_test = pd.read_csv('coupon_list_test.csv')
 cl_train = pd.read_csv('coupon_list_train.csv')
 sample_sub = pd.read_csv('sample_submission.csv')
 user_list = pd.read_csv('user_list.csv' )<create_dataframe>
dftrain = pd.read_csv('.. /input/train.csv')
 dftest = pd.read_csv('.. /input/test.csv')
 dftrain.head()
cd_train = cd_train.groupby(['USER_ID_hash', 'COUPON_ID_hash'])['PURCHASEID_hash'].count().to_frame().reset_index()<drop_column>
dftrain.any().isnull()
cl_train.drop(['VALIDFROM','VALIDEND'],axis=1,inplace=True)
 cl_test.drop(['VALIDFROM','VALIDEND'],axis=1,inplace=True)
 cl_train.VALIDPERIOD.fillna(180, inplace=True)
 cl_test.VALIDPERIOD.fillna(180, inplace=True)
 cl_train.fillna(1, inplace=True)
 cl_test.fillna(1, inplace=True )<create_dataframe>
dftrain['pickup_datetime'] = pd.to_datetime(dftrain['pickup_datetime'], format='%Y-%m-%d %H:%M:%S')
 dftrain['dropoff_datetime'] = pd.to_datetime(dftrain['dropoff_datetime'], format='%Y-%m-%d %H:%M:%S')
 dftest['pickup_datetime'] = pd.to_datetime(dftest['pickup_datetime'], format='%Y-%m-%d %H:%M:%S')
 dftrain.dtypes, dftest.dtypes
observation_data = turicreate.SFrame(cd_train)
 item_data = turicreate.SFrame(pd.concat([cl_train,cl_test]))<create_dataframe>
users = turicreate.SFrame(user_list[['USER_ID_hash']])
 items = turicreate.SFrame(cl_test[['COUPON_ID_hash']] )<choose_model_class>
dftrain = dftrain[dftrain['pickup_longitude'] >= -74.03]
dftrain = dftrain[dftrain['pickup_longitude'] <= -73.75]
dftrain = dftrain[dftrain['pickup_latitude'] <= 40.85]
dftrain = dftrain[dftrain['pickup_latitude'] >= 40.63]
dftrain = dftrain[dftrain['dropoff_longitude'] <= -73.75]
dftrain = dftrain[dftrain['dropoff_longitude'] >= -74.03]
dftrain = dftrain[dftrain['dropoff_latitude'] <= 40.85]
dftrain = dftrain[dftrain['dropoff_latitude'] >= 40.63]
cosine_model = turicreate.recommender.item_content_recommender.create(similarity_metrics='cosine', item_data=item_data, item_id='COUPON_ID_hash', observation_data=observation_data, target='PURCHASEID_hash', user_id='USER_ID_hash', item_data_transform='auto', verbose=True)
 jaccard_model = turicreate.recommender.item_content_recommender.create(similarity_metrics='jaccard', item_data=item_data, item_id='COUPON_ID_hash', observation_data=observation_data, target='PURCHASEID_hash', user_id='USER_ID_hash', item_data_transform='auto', verbose=True)
 pearson_model = turicreate.recommender.item_content_recommender.create(similarity_metrics='pearson', item_data=item_data, item_id='COUPON_ID_hash', observation_data=observation_data, target='PURCHASEID_hash', user_id='USER_ID_hash', item_data_transform='auto', verbose=True )<create_dataframe>
dftrain['month'] = dftrain['pickup_datetime'].dt.month
dftrain['week'] = dftrain['pickup_datetime'].dt.week
dftrain['weekday'] = dftrain['pickup_datetime'].dt.weekday
dftrain['hours'] = dftrain['pickup_datetime'].dt.hour
cosine_res = cosine_model.recommend(users, k=10, items=items ).to_dataframe()
jaccard_res = jaccard_model.recommend(users, k=10, items=items ).to_dataframe()
pearson_res = pearson_model.recommend(users, k=10, items=items ).to_dataframe()<create_dataframe>
dftest['month'] = dftest['pickup_datetime'].dt.month
dftest['week'] = dftest['pickup_datetime'].dt.week
dftest['weekday'] = dftest['pickup_datetime'].dt.weekday
dftest['hours'] = dftest['pickup_datetime'].dt.hour
def clean_prediction(row):
data = row.PURCHASED_COUPONS
data = str("".join(str(data)) [2:-2].replace("', '"," "))
return data<groupby>
dftrain['dist'] = np.sqrt(( dftrain['pickup_latitude']-dftrain['dropoff_latitude'])**2
+(dftrain['pickup_longitude']-dftrain['dropoff_longitude'])**2)
 dftest['dist'] = np.sqrt(( dftest['pickup_latitude']-dftest['dropoff_latitude'])**2
+(dftest['pickup_longitude']-dftest['dropoff_longitude'])**2 )
cosine_res = cosine_res.groupby('USER_ID_hash')['COUPON_ID_hash'].apply(list ).reset_index(name='PURCHASED_COUPONS')
 jaccard_res = jaccard_res.groupby('USER_ID_hash')['COUPON_ID_hash'].apply(list ).reset_index(name='PURCHASED_COUPONS')
 pearson_res = pearson_res.groupby('USER_ID_hash')['COUPON_ID_hash'].apply(list ).reset_index(name='PURCHASED_COUPONS' )<feature_engineering>
dftrain['store_and_fwd_flag'] = dftrain['store_and_fwd_flag'].astype('category' ).cat.codes
dftest['store_and_fwd_flag'] = dftest['store_and_fwd_flag'].astype('category' ).cat.codes
dftrain.head()
cosine_res['PURCHASED_COUPONS'] = cosine_res.apply(clean_prediction, axis=1)
 jaccard_res['PURCHASED_COUPONS'] = jaccard_res.apply(clean_prediction, axis=1)
 pearson_res['PURCHASED_COUPONS'] = pearson_res.apply(clean_prediction, axis=1 )<save_to_csv>
dftrain['store_and_fwd_flag'].value_counts()
cosine_res.to_csv('sub_cpp_turi_item_content_cosine.csv', index=False)
 jaccard_res.to_csv('sub_cpp_turi_item_content_jaccard.csv', index=False)
 pearson_res.to_csv('sub_cpp_turi_item_content_pearson.csv', index=False )<install_modules>
dftrain['pickup_datetime'] = dftrain['pickup_datetime'].astype(int)
 dftrain.head()
!pip install turicreate<import_modules>
dftest['pickup_datetime'] = dftest['pickup_datetime'].astype(int)
 dftest.head()
import os
import pandas as pd
import numpy as np
from zipfile import ZipFile
import turicreate<define_variables>
selection = ["vendor_id", "passenger_count", "pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","month", "weekday", "hours", "dist", "store_and_fwd_flag"]
X = dftrain[selection]
y = dftrain["trip_duration_log"]
X_test = dftest[selection]
ds_dir = '.. /input/coupon-purchase-prediction'<load_pretrained>
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split, cross_val_score
with ZipFile(os.path.join(ds_dir,"coupon_detail_train.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"coupon_list_test.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"coupon_list_train.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"coupon_visit_train.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"sample_submission.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"user_list.csv.zip"), 'r')as zipObj:
zipObj.extractall()<load_from_csv>
X_train, X_test, y_train, y_test = train_test_split(
 X, y, test_size=.2, random_state=42, stratify=y)
 X_train.shape, y_train.shape, X_test.shape, y_test.shape
cd_train = pd.read_csv('coupon_detail_train.csv')
 cl_test = pd.read_csv('coupon_list_test.csv')
 cl_train = pd.read_csv('coupon_list_train.csv')
 sample_sub = pd.read_csv('sample_submission.csv')
 user_list = pd.read_csv('user_list.csv' )<create_dataframe>
rf = RandomForestRegressor(n_estimators=100, min_samples_leaf=4, min_samples_split=15, n_jobs=-1 )
cd_train = cd_train.groupby(['USER_ID_hash', 'COUPON_ID_hash'])['PURCHASEID_hash'].count().to_frame().reset_index()<drop_column>
cl_train.drop(['VALIDFROM','VALIDEND'],axis=1,inplace=True)
 cl_test.drop(['VALIDFROM','VALIDEND'],axis=1,inplace=True)
 cl_train.VALIDPERIOD.fillna(180, inplace=True)
 cl_test.VALIDPERIOD.fillna(180, inplace=True)
 cl_train.fillna(1, inplace=True)
 cl_test.fillna(1, inplace=True )<create_dataframe>
rf.fit(X, y )
observation_data = turicreate.SFrame(cd_train)
 item_data = turicreate.SFrame(pd.concat([cl_train,cl_test]))<create_dataframe>
y_pred = rf.predict(X_test )
users = turicreate.SFrame(user_list[['USER_ID_hash']])
 items = turicreate.SFrame(cl_test[['COUPON_ID_hash']] )<choose_model_class>
my_submission = pd.DataFrame({'id': dftest.id, 'trip_duration': np.exp(y_pred)})
 my_submission.head()
<create_dataframe><EOS> 
my_submission.to_csv('submission.csv',index=False )