Kaggle competition: New York City Taxi Trip Duration
add_to_cart_counts = test_driver[['user_id', 'order_number']] \ 
.groupby(['user_id', 'order_number'])\ 
.size() \ 
.reset_index(name = 'total_cart_order')
 add_to_cart = pd.merge(
 test_driver[['user_id', 'order_number', 'product_id','add_to_cart_order']],
add_to_cart_counts,
on = ['user_id', 'order_number'],
how = 'inner' 
)
 add_to_cart['pct_add_to_cart'] = add_to_cart['add_to_cart_order'] / add_to_cart['total_cart_order']
last_order = add_to_cart.copy()
add_to_cart = add_to_cart[['user_id', 'product_id', 'pct_add_to_cart']] \ 
.groupby(['user_id', 'product_id'])\ 
.sum() ['pct_add_to_cart'] \ 
.reset_index(name = 'total_cart_order')
 add_to_cart = pd.merge(
 add_to_cart,
test_counts,
on = ['user_id', 'product_id'],
how = 'inner' 
)
 add_to_cart['avg_add_to_cart_order'] = add_to_cart['total_cart_order'] / add_to_cart['count_previous_orders']
last_order_driver = last_order[['user_id', 'product_id', 'order_number']] \ 
.groupby(['user_id', 'product_id'])\ 
.max() ['order_number'] \ 
.reset_index(name = 'order_number')
 last_order = pd.merge(
 last_order,
last_order_driver,
on = ['user_id', 'product_id', 'order_number'],
how = 'inner' 
)
 add_to_cart = pd.merge(
 add_to_cart,
last_order[['user_id', 'product_id', 'pct_add_to_cart']],
on = ['user_id', 'product_id'],
how = 'inner'
).drop('count_previous_orders', axis = 1)
 add_to_cart.head()<merge>
train = pd.read_csv(".. /input/train.csv" )
last_user_order = test_driver[['user_id', 'order_number']] \ 
.groupby(['user_id'])\ 
.max() ['order_number'] \ 
.reset_index(name = 'order_number_end')
 last_user_order['order_number_start'] = last_user_order['order_number_end'] - 2
last_user_order.loc[last_user_order.order_number_start < 0, 'order_number_start'] = 0
recent_three_orders = pd.merge(
 test_driver,
last_user_order,
on = 'user_id',
how = 'inner' 
)
 recent_three_orders = recent_three_orders[ 
(recent_three_orders.order_number >= recent_three_orders.order_number_start)& 
(recent_three_orders.order_number <= recent_three_orders.order_number_end)
 ]
recent_three_orders = recent_three_orders[['user_id', 'product_id']] \ 
.groupby(['user_id', 'product_id'])\ 
.size() \ 
.reset_index(name = 'num_orders_last_three')
 recent_three_orders<merge>
print(train.store_and_fwd_flag.value_counts() )
test = pd.merge(
 test,
test_counts,
on = ['user_id'],
how = 'inner' 
)
 test = pd.merge(
 test,
test_last_order,
left_on = ['user_id', 'product_id', 'order_number'],
right_on = ['user_id', 'product_id', 'order_number_current'],
how = 'inner' 
)
 test = pd.merge(
 test,
products,
on = 'product_id',
how = 'inner'
).drop('product_name', axis = 'columns')
 test = pd.merge(
 test,
add_to_cart,
on = ['user_id', 'product_id'],
how = 'inner' 
)
 test = pd.merge(
 test,
product_rank,
on = ['product_id', 'order_dow', 'order_hour_of_day'],
how = 'inner' 
)
 test = pd.merge(
 test,
product_rank_day,
on = ['product_id', 'order_dow'],
how = 'inner' 
)
 test = pd.merge(
 test,
product_rank_hour,
on = ['product_id', 'order_hour_of_day'],
how = 'inner' 
)
 test = pd.merge(
 test,
product_rank_overall,
on = ['product_id'],
how = 'inner' 
)
 test = pd.merge(
 test,
recent_three_orders,
on = ['user_id', 'product_id'],
how = 'left' 
)
 test['num_orders_last_three'][test.num_orders_last_three.isnull() ] = 0
test = test.drop(
 [
'eval_set',
'order_number_current',
'order_number_last',
'order_id'
],
axis = 1
)<merge>
def drop_duration(df):
df.loc[df.trip_duration < trip_duration_min,'trip_duration'] = np.nan
value=df.trip_duration.min()
df.trip_duration.fillna(value=value, inplace=True)
 df.loc[df.trip_duration > trip_duration_max,'trip_duration'] = np.nan
value=df.trip_duration.max()
df.trip_duration.fillna(value=value, inplace=True)
 return
poly_test = test[
[
'num_orders_last_three',
'count_previous_orders',
'total_cart_order',
'days_since_last_order'
]
]
p_test = PolynomialFeatures(degree = 2 ).fit(poly_test)
 poly_test = pd.DataFrame(
 data = p_test.transform(poly_test),
columns = p_test.get_feature_names(poly_test.columns)
)
 poly_test['user_id'] = test['user_id']
poly_test['order_number'] = test['order_number']
poly_test['product_id'] = test['product_id']
test = pd.merge(
 test,
poly_test,
on = [
'user_id',
'order_number',
'product_id',
'num_orders_last_three',
'count_previous_orders',
'total_cart_order',
'days_since_last_order'
],
how = 'inner'
).drop('1', axis = 1)
 test.shape<drop_column>
drop_duration(train )
test = test[
[
'user_id',
'product_id',
'order_number',
'order_dow',
'order_hour_of_day',
'days_since_prior_order',
'days_agg',
'count_previous_orders',
'days_agg_last',
'days_since_last_order',
'pct_add_to_cart',
'total_cart_order',
'avg_add_to_cart_order',
'num_orders_last_three',
'aisle_id',
'department_id',
'count_day_hour',
'rank_macro_day_hour',
'rank_product_day_hour',
'count_day',
'rank_macro_day',
'rank_product_day',
'count_hour',
'rank_macro_hour',
'rank_product_hour',
'count_overall',
'rank_macro_overall',
'num_orders_last_three^2',
'num_orders_last_three count_previous_orders',
'num_orders_last_three total_cart_order',
'num_orders_last_three days_since_last_order',
'count_previous_orders^2',
'count_previous_orders total_cart_order',
'count_previous_orders days_since_last_order',
'total_cart_order^2',
'total_cart_order days_since_last_order',
'days_since_last_order^2'
]
]<merge>
def conf_int_duration(df):

 conf_int_duration = np.percentile(df.trip_duration, [2.5,97.5])
 print('
Confidental interval trip_duration: {}'.format(conf_int_duration))
df.loc[df.trip_duration < conf_int_duration[0],'trip_duration'] = np.nan
value=df.trip_duration.min()
df.trip_duration.fillna(value=value, inplace=True)
 df.loc[df.trip_duration > conf_int_duration[1],'trip_duration'] = np.nan
value=df.trip_duration.max()
df.trip_duration.fillna(value=value, inplace=True)
 print("Trip_duration describe past drop:
",df.trip_duration.describe())
 return
test['prediction'] = lgb_model.predict_proba(test)[:,1]
submit_staging = test[['user_id', 'product_id', 'order_number']][test.prediction >=.7]
submit_staging = pd.merge(
 submit_staging,
orders[['user_id', 'order_number', 'order_id']],
on = ['user_id', 'order_number'],
how = 'inner' 
)
 test_left_side = orders[orders.eval_set == 'test']
submit_staging2 = pd.merge(
 test_left_side[['user_id', 'order_id']],
submit_staging,
on = ['user_id', 'order_id'],
how = 'left' 
)
 submit_staging3 = submit_staging2[['order_id', 'product_id']]
submit_staging3['product_id'][submit_staging3.product_id.isnull() ] = 0
submit_staging3 = submit_staging3.astype(int)
 submit_staging3 = submit_staging3.astype(str)
 submit = submit_staging3 \ 
.groupby('order_id')['product_id'] \ 
.apply(' '.join)\ 
.reset_index(name = 'products')
 submit.loc[submit.products == '0', 'products'] = 'None'
submit.to_csv('submit.csv', index = False )<import_modules>
test = pd.read_csv(".. /input/test.csv")
 test.info()
import numpy as np
import pandas as pd
import xgboost as xgb
from sklearn import metrics, model_selection<load_from_csv>
result['pickup_longitude'] = result.pickup_longitude.round(5)
 result['pickup_latitude'] = result.pickup_latitude.round(5)
 result.loc[result.pickup_latitude < pickup_lat_min, 'pickup_latitude'] = np.nan
value=result.pickup_latitude.min()
result.pickup_latitude.fillna(value=value, inplace=True)
 result.loc[result.pickup_latitude > pickup_lat_max, 'pickup_latitude'] = np.nan
value=result.pickup_latitude.max()
result.pickup_latitude.fillna(value=value, inplace=True)
 result.loc[result.pickup_longitude < pickup_long_min, 'pickup_longitude'] = np.nan
value=result.pickup_longitude.min()
result.pickup_longitude.fillna(value=value, inplace=True)
 result.loc[result.pickup_longitude > pickup_long_max, 'pickup_longitude'] = np.nan
value=result.pickup_longitude.max()
result.pickup_longitude.fillna(value=value, inplace=True)
 result['dropoff_longitude'] = result.dropoff_longitude.round(5)
 result['dropoff_latitude'] = result.dropoff_latitude.round(5)
 result.loc[result.dropoff_latitude < dropoff_lat_min, 'dropoff_latitude'] = np.nan
value=result.dropoff_latitude.min()
result.dropoff_latitude.fillna(value=value, inplace=True)
 result.loc[result.dropoff_latitude > dropoff_lat_max, 'dropoff_latitude'] = np.nan
value=result.dropoff_latitude.max()
result.dropoff_latitude.fillna(value=value, inplace=True)
 result.loc[result.dropoff_longitude < dropoff_long_min, 'dropoff_longitude'] = np.nan
value=result.dropoff_longitude.min()
result.dropoff_longitude.fillna(value=value, inplace=True)
 result.loc[result.dropoff_longitude > dropoff_long_max, 'dropoff_longitude'] = np.nan
value=result.dropoff_longitude.max()
result.dropoff_longitude.fillna(value=value, inplace=True)
 print(result.describe() )
data_path = ".. /input/"
orders_df = pd.read_csv(data_path + "orders.csv", usecols=["order_id","user_id","order_number"] )<load_from_csv>
AVG_EARTH_RADIUS = 6371
def haversine(df, miles=True):

 lat1, lng1, lat2, lng2 =(df.pickup_latitude[:],
df.pickup_longitude[:],
df.dropoff_latitude[:],
df.dropoff_longitude[:])
 lat1, lng1, lat2, lng2 = map(np.radians,(lat1, lng1, lat2, lng2))
lat = lat2 - lat1
lng = lng2 - lng1
d = np.sin(lat*0.5)**2 + np.cos(lat1)* np.cos(lat2)* np.sin(lng*0.5)**2
h = 2 * AVG_EARTH_RADIUS * np.arcsin(np.sqrt(d))
if miles:
df['trip_distance'] = h * 0.621371
df['trip_distance'] = df.trip_distance.round(2)
 print(df.trip_distance.describe())
 return
else:
df['trip_distance'] = h
df['trip_distance'] = df.trip_distance.round(2)
 print(df.trip_distance.describe())
 return
aisles=pd.read_csv('.. /input/aisles.csv')
 departments=pd.read_csv('.. /input/departments.csv')
 orders=pd.read_csv('.. /input/orders.csv')
 orderp=pd.read_csv('.. /input/order_products__prior.csv')
 ordert=pd.read_csv('.. /input/order_products__train.csv')
 products=pd.read_csv('.. /input/products.csv' )<merge>
haversine(result, miles=True )
goods = pd.merge(left=pd.merge(left=products, right=departments, how='left'), right=aisles, how='left')
 goods.product_name = goods.product_name.str.replace(' ', '_' ).str.lower()
goods.head()<merge>
def arrays_bearing(df):

 lats1, lngs1, lats2, lngs2 =(df['pickup_latitude'][:],
df['pickup_longitude'][:],
df['dropoff_latitude'][:],
df['dropoff_longitude'][:])
 lats1_rads = np.radians(lats1)
 lats2_rads = np.radians(lats2)
 lngs_delta_rads = np.radians(lngs2 - lngs1)
 y = np.sin(lngs_delta_rads)* np.cos(lats2_rads)
 x = np.cos(lats1_rads)* np.sin(lats2_rads)- \
np.sin(lats1_rads)* np.cos(lats2_rads)* np.cos(lngs_delta_rads)
 df['bearing'] = np.degrees(np.arctan2(y, x))
df['bearing'] = df.bearing.round(0)
 print(df.bearing.describe())
 return
prior_df = pd.read_csv(data_path + "order_products__prior.csv")
 prior_df = pd.merge(prior_df, orders_df, how="inner", on="order_id")
 prior_grouped_df = prior_df.groupby("user_id")["order_number"].aggregate("max" ).reset_index()
prior_df_latest = pd.merge(prior_df, prior_grouped_df, how="inner", on=["user_id", "order_number"])
 prior_df_latest = prior_df_latest[["user_id", "product_id", "reordered"]]
prior_df_latest.columns = ["user_id", "product_id", "reordered_latest"]
prior_df = prior_df.groupby(["user_id","product_id"])["reordered"].aggregate(["count", "sum"] ).reset_index()
prior_df.columns = ["user_id", "product_id", "reordered_count", "reordered_sum"]
prior_df = pd.merge(prior_df, prior_df_latest, how="left", on=["user_id","product_id"])
 prior_df.head()<merge>
result.drop('dropoff_datetime', axis=1, inplace=True )
orders_df.drop(["order_number"],axis=1,inplace=True)
 train_df = pd.read_csv(data_path + "order_products__train.csv", usecols=["order_id"])
 train_df = train_df.groupby("order_id" ).aggregate("count" ).reset_index()
test_df = pd.read_csv(data_path + "sample_submission.csv", usecols=["order_id"])
 train_df = pd.merge(train_df, orders_df, how="inner", on="order_id")
 test_df = pd.merge(test_df, orders_df, how="inner", on="order_id")
 print(train_df.shape, test_df.shape )<merge>
result['pickup_datetime'] = pd.to_datetime(result.pickup_datetime )
train_df = pd.merge(train_df, prior_df, how="inner", on="user_id")
 test_df = pd.merge(test_df, prior_df, how="inner", on="user_id")
 del prior_df, prior_grouped_df, prior_df_latest
print(train_df.shape, test_df.shape )<merge>
result['days_in_month'] = result['pickup_datetime'][:].dt.days_in_month
products_df = pd.read_csv(data_path + "products.csv", usecols=["product_id", "aisle_id", "department_id"])
 train_df = pd.merge(train_df, products_df, how="inner", on="product_id")
 test_df = pd.merge(test_df, products_df, how="inner", on="product_id")
 del products_df
print(train_df.shape, test_df.shape )<merge>
result['weekday'] = result['pickup_datetime'].dt.weekday
train_y_df = pd.read_csv(data_path + "order_products__train.csv", usecols=["order_id", "product_id", "reordered"])
 train_y_df = pd.merge(train_y_df, orders_df, how="inner", on="order_id")
 train_y_df = train_y_df[["user_id", "product_id", "reordered"]]
train_df = pd.merge(train_df, train_y_df, how="left", on=["user_id", "product_id"])
 train_df["reordered"].fillna(0, inplace=True)
 print(train_df.shape)
 del train_y_df<prepare_x_and_y>
result['hour'] = result['pickup_datetime'][:].dt.hour
train_y = train_df.reordered.values
out_df = test_df[["order_id", "product_id"]]
train_df = np.array(train_df.drop(["order_id", "user_id", "reordered"], axis=1))
test_df = np.array(test_df.drop(["order_id", "user_id"], axis=1))
print(train_df.shape, test_df.shape )<train_model>
result['minute'] = result['pickup_datetime'][:].dt.minute
def runXGB(train_X, train_y, test_X, test_y=None, feature_names=None, seed_val=0):
params = {}
params["objective"] = "binary:logistic"
params['eval_metric'] = 'logloss'
params["eta"] = 0.05
params["subsample"] = 0.7
params["min_child_weight"] = 10
params["colsample_bytree"] = 0.7
params["max_depth"] = 8
params["silent"] = 1
params["seed"] = seed_val
num_rounds = 100
plst = list(params.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 if test_y is not None:
xgtest = xgb.DMatrix(test_X, label=test_y)
 watchlist = [(xgtrain,'train'),(xgtest, 'test')]
model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds=50, verbose_eval=10)
 else:
xgtest = xgb.DMatrix(test_X)
 model = xgb.train(plst, xgtrain, num_rounds)
 pred_test_y = model.predict(xgtest)
 return pred_test_y<data_type_conversions>
result['month'] = result['pickup_datetime'][:].dt.month
pred = runXGB(train_df, train_y, test_df)
 del train_df, test_df
cutoff = 0.2
pred[pred>=cutoff] = 1
pred[pred<cutoff] = 0
out_df["Pred"] = pred
out_df = out_df.ix[out_df["Pred"].astype('int')==1]<categorify>
result.drop(['pickup_datetime', 'store_and_fwd_flag', 'id'], axis=1, inplace=True )
def merge_products(x):
return " ".join(list(x.astype('str')))
 out_df = out_df.groupby("order_id")["product_id"].aggregate(merge_products ).reset_index()
out_df.columns = ["order_id", "products"]<save_to_csv>
test.drop('trip_duration', axis=1, inplace=True )
sub_df = pd.read_csv(data_path + "sample_submission.csv", usecols=["order_id"])
 sub_df = pd.merge(sub_df, out_df, how="left", on="order_id")
 sub_df["products"].fillna("None", inplace=True)
 sub_df.to_csv("xgb_starter_3450.csv", index=False )<import_modules>
train.trip_duration =(train['trip_duration']+1 ).apply(np.log)
 trip_duration = train.trip_duration.values
import numpy as np
import pandas as pd
import xgboost as xgb
from sklearn.preprocessing import LabelEncoder
import lightgbm as lgb
import gc
from sklearn.linear_model import LinearRegression
import random
import datetime as dt
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout, BatchNormalization
from keras.layers.advanced_activations import PReLU
from keras.layers.noise import GaussianDropout
from keras.optimizers import Adam
from keras.wrappers.scikit_learn import KerasRegressor
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import Imputer 
<load_from_csv>
train.drop('trip_duration', axis=1, inplace=True )
prox=pd.read_csv('.. /input/properties_2016.csv')
<data_type_conversions>
X = train.values
for c in prox.columns:
if(c=='airconditioningtypeid' or c=='taxdelinquencyflag' or c=='poolcnt' or c=='regionidcity' or c=='fireplacecnt' or c=='propertyzoningdesc' or c=='propertycountylandusecode'):
prox[c]=prox[c].fillna(-1)
 else:
prox[c]=prox[c].fillna(prox[c].median())
 prox.head()<categorify>
from sklearn.preprocessing import MinMaxScaler
lbl = LabelEncoder()
for c in prox.columns:
prox[c]=prox[c].fillna(-1)
 if prox[c].dtype == 'object':
lbl.fit(list(prox[c].values))
prox[c] = lbl.transform(list(prox[c].values))
prox.head()<feature_engineering>
scaler = MinMaxScaler().fit(X )
prox['N-life'] = 2018 - prox['yearbuilt']
prox['N-LivingAreaError'] = prox['calculatedfinishedsquarefeet']/prox['finishedsquarefeet12']
prox['N-LivingAreaProp'] = prox['calculatedfinishedsquarefeet']/prox['lotsizesquarefeet']
prox['N-LivingAreaProp2'] = prox['finishedsquarefeet12']/prox['finishedsquarefeet15']
prox['N-ExtraSpace'] = prox['lotsizesquarefeet'] - prox['calculatedfinishedsquarefeet']
prox['N-ExtraSpace-2'] = prox['finishedsquarefeet15'] - prox['finishedsquarefeet12']
prox['N-TotalRooms'] = prox['bathroomcnt']*prox['bedroomcnt']
prox['N-AvRoomSize'] = prox['calculatedfinishedsquarefeet']/prox['roomcnt']
prox['N-ExtraRooms'] = prox['roomcnt'] - prox['N-TotalRooms']
prox['N-ValueProp'] = prox['structuretaxvaluedollarcnt']/prox['landtaxvaluedollarcnt']
prox['N-GarPoolAC'] =(( prox['garagecarcnt']>0)&(prox['pooltypeid10']>0)&(prox['airconditioningtypeid']!=5)) *1
prox["N-location"] = prox["latitude"] + prox["longitude"]
prox["N-location-2"] = prox["latitude"]*prox["longitude"]
prox["N-location-2round"] = prox["N-location-2"].round(-4)
 prox["N-latitude-round"] = prox["latitude"].round(-4)
 prox["N-longitude-round"] = prox["longitude"].round(-4)
 prox.head() 
<feature_engineering>
X_scaled = scaler.transform(X )
prox['N-ValueRatio'] = prox['taxvaluedollarcnt']/prox['taxamount']
prox['N-TaxScore'] = prox['taxvaluedollarcnt']*prox['taxamount']
prox["N-taxdelinquencyyear-2"] = prox["taxdelinquencyyear"] ** 2
prox["N-taxdelinquencyyear-3"] = prox["taxdelinquencyyear"] ** 3
prox['N-life'] = 2018 - prox['taxdelinquencyyear'] 
<count_values>
y = trip_duration
zip_count = prox['regionidzip'].value_counts().to_dict()
prox['N-zip_count'] = prox['regionidzip'].map(zip_count)
 city_count = prox['regionidcity'].value_counts().to_dict()
prox['N-city_count'] = prox['regionidcity'].map(city_count)
 region_count = prox['regionidcounty'].value_counts().to_dict()
prox['N-county_count'] = prox['regionidcounty'].map(city_count)
<feature_engineering>
from sklearn.model_selection import train_test_split
prox['N-ACInd'] =(prox['airconditioningtypeid']!=5)*1
prox['N-HeatInd'] =(prox['heatingorsystemtypeid']!=13)*1
prox['N-PropType'] = prox.propertylandusetypeid.replace({31 : "Mixed", 46 : "Other", 47 : "Mixed", 246 : "Mixed", 247 : "Mixed", 248 : "Mixed", 260 : "Home", 261 : "Home", 262 : "Home", 263 : "Home", 264 : "Home", 265 : "Home", 266 : "Home", 267 : "Home", 268 : "Home", 269 : "Not Built", 270 : "Home", 271 : "Home", 273 : "Home", 274 : "Other", 275 : "Home", 276 : "Home", 279 : "Home", 290 : "Not Built", 291 : "Not Built" } )<feature_engineering>
from sklearn.model_selection import train_test_split
prox["N-structuretaxvaluedollarcnt-2"] = prox["structuretaxvaluedollarcnt"] ** 2
prox["N-structuretaxvaluedollarcnt-3"] = prox["structuretaxvaluedollarcnt"] ** 3
group = prox.groupby('regionidcity')['structuretaxvaluedollarcnt'].aggregate('mean' ).to_dict()
prox['N-Avg-structuretaxvaluedollarcnt'] = prox['regionidcity'].map(group)
 prox['N-Dev-structuretaxvaluedollarcnt'] = abs(( prox['structuretaxvaluedollarcnt'] - prox['N-Avg-structuretaxvaluedollarcnt'])) /prox['N-Avg-structuretaxvaluedollarcnt'] 
<load_from_csv>
X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, random_state=96 )
print("
Reading data from disk...")
 prop = prox
train = pd.read_csv(".. /input/train_2016_v2.csv" )<prepare_x_and_y>
X_train,X_val,y_train,y_val = \
train_test_split(X_train,y_train, random_state=144 )
print("
Processing data for LightGBM...")
 for c, dtype in zip(prop.columns, prop.dtypes):	
if dtype == np.float64:
prop[c] = prop[c].astype(np.float32)
 df_train = train.merge(prop, how='left', on='parcelid')
 df_train.fillna(df_train.median() ,inplace = True)
 x_train = df_train.drop(['parcelid', 'logerror', 'transactiondate', 'propertyzoningdesc',
'propertycountylandusecode', 'fireplacecnt', 'fireplaceflag'], axis=1)
 y_train = df_train['logerror'].values
print(x_train.shape, y_train.shape)
 train_columns = x_train.columns
for c in x_train.dtypes[x_train.dtypes == object].index.values:
x_train[c] =(x_train[c] == True)
 del df_train; gc.collect()
x_train = x_train.values.astype(np.float32, copy=False)
 d_train = lgb.Dataset(x_train, label=y_train )<init_hyperparams>
import xgboost as xgb
params = {}
params['max_bin'] = 10
params['learning_rate'] = 0.0021
params['boosting_type'] = 'gbdt'
params['objective'] = 'regression'
params['metric'] = 'l1'
params['sub_feature'] = 0.345
params['bagging_fraction'] = 0.85
params['bagging_freq'] = 40
params['num_leaves'] = 512
params['min_data'] = 500
params['min_hessian'] = 0.05
params['verbose'] = 0
params['feature_fraction_seed'] = 2
params['bagging_seed'] = 3
np.random.seed(0)
 random.seed(0)
<load_from_csv>
train_xgb = xgb.DMatrix(X_train, label=y_train)
 cv_xgb = xgb.DMatrix(X_val , label=y_val)
 evallist = [(train_xgb, 'train'),(cv_xgb, 'valid')]
print("
Fitting LightGBM model...")
 clf = lgb.train(params, d_train, 430)
 del d_train; gc.collect()
del x_train; gc.collect()
print("
Prepare for LightGBM prediction...")
 print(" Read sample file...")
 sample = pd.read_csv('.. /input/sample_submission.csv')
 print("...")
 sample['parcelid'] = sample['ParcelId']
print(" Merge with property data...")
 df_test = sample.merge(prop, on='parcelid', how='left')
 print("...")
 del sample, prop; gc.collect()
print("...")
 x_test = df_test[train_columns]
print("...")
 del df_test; gc.collect()
print(" Preparing x_test...")
 for c in x_test.dtypes[x_test.dtypes == object].index.values:
x_test[c] =(x_test[c] == True)
 print("...")
 x_test = x_test.values.astype(np.float32, copy=False)
 print("
Start LightGBM prediction...")
 p_test = clf.predict(x_test)
 del x_test; gc.collect()
print("
Unadjusted LightGBM predictions:")
 print(pd.DataFrame(p_test ).head())
<load_pretrained>
param = {'max_depth':10,
'objective':'reg:linear',
'eta' :.1,
'subsample':0.3,
'lambda ' :6,
'colsample_bytree ':0.3,
'colsample_bylevel':1,
'min_child_weight': 20,
'nthread': -1}
model = xgb.train(param, train_xgb, num_boost_round=1000, evals = evallist,
early_stopping_rounds=50, maximize=False,
verbose_eval=50)
 print("score = {:.5f}, n_boost_round = {}".format(model.best_score,
model.best_iteration))
print("
Re-reading properties file...")
 properties = prox 
<categorify>
X_real_test = test.values
print("
Processing data for XGBoost...")
 for c in properties.columns:
properties[c]=properties[c].fillna(-1)
 if properties[c].dtype == 'object':
lbl = LabelEncoder()
lbl.fit(list(properties[c].values))
properties[c] = lbl.transform(list(properties[c].values))
train_df = train.merge(properties, how='left', on='parcelid')
 x_train = train_df.drop(['parcelid', 'logerror','transactiondate'], axis=1)
 x_test = properties.drop(['parcelid'], axis=1)
 print('Shape train: {}
Shape test: {}'.format(x_train.shape, x_test.shape))<train_model>
X_test_scaled = scaler.transform(X_real_test )
print("
Setting up data for XGBoost...")
 xgb_params = {
'eta': 0.037,
'max_depth': 5,
'subsample': 0.80,
'objective': 'reg:linear',
'eval_metric': 'mae',
'lambda': 0.8,
'alpha': 0.4,
'base_score': y_mean,
'silent': 1
}
dtrain = xgb.DMatrix(x_train, y_train)
 dtest = xgb.DMatrix(x_test)
 num_boost_rounds = 250
print("num_boost_rounds="+str(num_boost_rounds))
print("
Training XGBoost...")
 model = xgb.train(dict(xgb_params, silent=1), dtrain, num_boost_round=num_boost_rounds)
 print("
Predicting with XGBoost...")
 xgb_pred1 = model.predict(dtest)
 print("
First XGBoost predictions:")
 print(pd.DataFrame(xgb_pred1 ).head())
<train_model>
test_xgb = xgb.DMatrix(X_test_scaled)
 y_pred = model.predict(test_xgb )
print("
Setting up data for XGBoost...")
 xgb_params = {
'eta': 0.033,
'max_depth': 6,
'subsample': 0.80,
'objective': 'reg:linear',
'eval_metric': 'mae',
'base_score': y_mean,
'silent': 1
}
num_boost_rounds = 150
print("num_boost_rounds="+str(num_boost_rounds))
print("
Training XGBoost again...")
 model = xgb.train(dict(xgb_params, silent=1), dtrain, num_boost_round=num_boost_rounds)
 print("
Predicting with XGBoost again...")
 xgb_pred2 = model.predict(dtest)
 print("
Second XGBoost predictions:")
 print(pd.DataFrame(xgb_pred2 ).head())
<prepare_output>
y_pred = np.exp(y_pred[:])- 1
xgb_pred = XGB1_WEIGHT*xgb_pred1 +(1-XGB1_WEIGHT)*xgb_pred2
print("
Combined XGBoost predictions:")
 print(pd.DataFrame(xgb_pred ).head() )<drop_column>
submission = pd.read_csv('.. /input/sample_submission.csv', index_col=0, header=0 )
del train_df
del x_train
del x_test
del properties
del dtest
del dtrain
del xgb_pred1
del xgb_pred2
gc.collect()<load_from_csv>
submission.trip_duration = y_pred.round(0)
 submission.head(10 )
<feature_engineering><EOS> 
submission.to_csv('submission.csv' )