Kaggle competition: New York City Taxi Trip Duration
<SOS> metric: RMSLE Kaggle data source: new-york-city-taxi-trip-duration<categorify>
import os
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import seaborn as sns
import math
from datetime import datetime
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond', 'HouseStyle',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1','Neighborhood', 'SaleCondition',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope','Condition1','Condition2',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond','GarageType','SaleType','BldgType')
 for c in cols:
lb = LabelEncoder()
lb.fit(list(data[c].values))
data[c] = lb.transform(list(data[c].values))<categorify>
%matplotlib inline
sns.set({'figure.figsize':(16,8), 'axes.titlesize':15, 'axes.labelsize':10} )
data = pd.get_dummies(data )<feature_engineering>
FILEPATH = os.path.join(".. ", "input")
 TRAINPATH = os.path.join(FILEPATH, "train.csv")
 TESTPATH = os.path.join(FILEPATH, "test.csv" )
data['Total_Bathrooms'] = data['FullBath'] + 0.5 * data['HalfBath'] + data['BsmtFullBath'] + 0.5 * data['BsmtHalfBath']
data['GarageScale'] = data['GarageCars'] * data['GarageArea']
data['GarageOrdinal'] = data['GarageQual'] + data['GarageCond']+data['GarageType']
data['GarageState'] = data['GarageFinish'] + data['GarageYrBlt'] + data['PavedDrive']
data['Porchtotal'] = data['OpenPorchSF'] + data['EnclosedPorch'] + data['3SsnPorch'] + data['ScreenPorch']
data['Extertotal'] = data['ExterQual'] + data['ExterCond']
data['KitchenCombined'] = data['KitchenQual'] * data['KitchenAbvGr']
data['FireplaceCombined'] = data['FireplaceQu'] * data['Fireplaces']
data['BsmtOrdinal'] = data['BsmtQual'] + data['BsmtCond']
data['BsmtFinishedAll'] = data['BsmtFinSF1'] + data['BsmtFinSF2']
data['AllFlrSF'] = data['1stFlrSF'] + data['2ndFlrSF']
data['OverallCombined'] = data['OverallQual'] + data['OverallCond']
data['TotalSF'] = data['AllFlrSF'] + data['TotalBsmtSF']
data['YrBltAndRemod'] = data["YearRemodAdd"] + data['YearBuilt']
data['roomtotal'] = data['BedroomAbvGr'] + data['KitchenAbvGr']+ data['TotRmsAbvGrd']
data['SaleState'] = data['SaleType'] + data['SaleCondition']
data['HouseComment'] = data['BldgType'] + data['HouseStyle'] + data['OverallCond']
data['Condition'] = data['Condition1'] + data['Condition2'] + data['Neighborhood']<feature_engineering>
df = pd.read_csv(TRAINPATH, index_col=0)
 df.head()
def logs(res, ls):
m = res.shape[1]
for l in ls:
res = res.assign(newcol=pd.Series(np.log(1.01+res[l])).values)
 res.columns.values[m] = l + '_log'
m += 1
return res
log_features = ['LotFrontage','LotArea','MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF',
'TotalBsmtSF','1stFlrSF','2ndFlrSF','LowQualFinSF','GrLivArea',
'BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr',
'TotRmsAbvGrd','Fireplaces','GarageCars','GarageArea','WoodDeckSF','OpenPorchSF',
'EnclosedPorch','3SsnPorch','ScreenPorch','PoolArea','MiscVal','YearRemodAdd','Total_Bathrooms'
,'GarageScale','GarageOrdinal','Porchtotal','Extertotal','KitchenCombined','FireplaceCombined','BsmtOrdinal',
'BsmtFinishedAll','AllFlrSF','OverallCombined','TotalSF','YrBltAndRemod','roomtotal','SaleState','HouseComment','Condition']
data = logs(data, log_features )<feature_engineering>
df_test = pd.read_csv(TESTPATH)
 df_test.head()
<drop_column>
df = df[df.passenger_count != 0]
train_data = data[:1460]
test_data = data[1460:].drop(['SalePrice'],axis = 1 )<prepare_x_and_y>
df.duplicated().sum()
y = np.log1p(train_data['SalePrice'])
 x = train_data.drop(['SalePrice'],axis=1 )<split>
df = df.drop_duplicates()
df.duplicated().sum()
xtrain ,xtest ,ytrain ,ytest = train_test_split(x,y,test_size = 0.3, random_state = 4200 )<predict_on_test>
df.isna().sum()
la = Lasso(alpha=0.1,max_iter=500)
 la.fit(x,y)
 pred1 = np.expm1(la.predict(test_data))<predict_on_test>
df = df[(df['trip_duration'] > 60)&(df['trip_duration'] < 3600 * 6)]
R = Ridge(alpha=1.0,max_iter=500)
 R.fit(x,y)
 pred2 = np.expm1(R.predict(test_data))<choose_model_class>
df['store_and_fwd_flag'] = df['store_and_fwd_flag'].astype('category' ).cat.codes
df_test['store_and_fwd_flag'] = df_test['store_and_fwd_flag'].astype('category' ).cat.codes
xgbr = XGBRegressor(booster='gbtree'
, colsample_bylevel=1
, colsample_bynode=1
, colsample_bytree=0.6
, gamma=0
, importance_type='gain'
, learning_rate=0.01
, max_delta_step=0
, max_depth= 3
, min_child_weight=1.5
, n_estimators=5400
, n_jobs=-1
, nthread=None
, objective='reg:squarederror'
, reg_alpha=0.3
, reg_lambda=0.7
, scale_pos_weight=1
, silent=None
, subsample=0.6
, verbosity=1)
 xgbr.fit(x,y)
 pred3 = np.expm1(xgbr.predict(test_data))<choose_model_class>
from sklearn.decomposition import PCA
lgmb = LGBMRegressor(objective='regression',
boosting_type='gbdt',
num_leaves= 5,
max_depth = 3,
learning_rate=0.01,
n_estimators=6300,
subsample_for_bin=100,
class_weight=None,
min_split_gain=0.0,
min_child_weight=1.3,
min_child_samples=3,
subsample=0.1,
subsample_freq=0,
colsample_bytree=0.6,
reg_alpha=0.1,
reg_lambda=0.6,
random_state=None,
n_jobs= -1,
silent=True,
importance_type='gain')
 lgmb.fit(x,y)
 pred4 = np.expm1(lgmb.predict(test_data))<choose_model_class>
coords = np.vstack(( df[['pickup_latitude', 'pickup_longitude']].values,
df[['dropoff_latitude', 'dropoff_longitude']].values,
df_test[['pickup_latitude', 'pickup_longitude']].values,
df_test[['dropoff_latitude', 'dropoff_longitude']].values))
pca = PCA().fit(coords)
 df['pickup_pca0'] = pca.transform(df[['pickup_latitude', 'pickup_longitude']])[:, 0]
df['pickup_pca1'] = pca.transform(df[['pickup_latitude', 'pickup_longitude']])[:, 1]
df['dropoff_pca0'] = pca.transform(df[['dropoff_latitude', 'dropoff_longitude']])[:, 0]
df['dropoff_pca1'] = pca.transform(df[['dropoff_latitude', 'dropoff_longitude']])[:, 1]
df_test['pickup_pca0'] = pca.transform(df_test[['pickup_latitude', 'pickup_longitude']])[:, 0]
df_test['pickup_pca1'] = pca.transform(df_test[['pickup_latitude', 'pickup_longitude']])[:, 1]
df_test['dropoff_pca0'] = pca.transform(df_test[['dropoff_latitude', 'dropoff_longitude']])[:, 0]
df_test['dropoff_pca1'] = pca.transform(df_test[['dropoff_latitude', 'dropoff_longitude']])[:, 1]
gbrt = GradientBoostingRegressor(alpha=0.5, ccp_alpha=0.0,
criterion='friedman_mse',
init=None, learning_rate=0.01,
loss='huber', max_depth=3,
max_leaf_nodes=5,
min_impurity_decrease=0.0,
min_impurity_split=None,
min_samples_leaf=10,
min_samples_split=3,
min_weight_fraction_leaf=0.0,
n_estimators=3350,
n_iter_no_change=None,
random_state=5,subsample=0.6,
tol=0.0001,
validation_fraction=0.1,
verbose=0)
 gbrt.fit(x,y)
 pred5 = np.expm1(gbrt.predict(test_data))<choose_model_class>
df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'])
 df['dropoff_datetime'] = pd.to_datetime(df['dropoff_datetime'])
 df_test['pickup_datetime'] = pd.to_datetime(df_test['pickup_datetime'] )
E = ElasticNet(alpha=0.1,l1_ratio=0.1)
 E.fit(x,y)
 pred6 = np.expm1(E.predict(test_data))<define_variables>
df['hour'] = df.pickup_datetime.dt.hour
df['day'] = df.pickup_datetime.dt.dayofweek
df['month'] = df.pickup_datetime.dt.month
df_test['hour'] = df_test.pickup_datetime.dt.hour
df_test['day'] = df_test.pickup_datetime.dt.dayofweek
df_test['month'] = df_test.pickup_datetime.dt.month
pred = 0.0* pred1 + 0.2* pred2 + 0.37*pred3 + 0.38*pred4 +0.00* pred5+0.05* pred6<save_to_csv>
df['distance2'] = np.sqrt(( df['pickup_pca0']-df['dropoff_pca0'])**2
+(df['pickup_pca1']-df['dropoff_pca1'])**2)
 df_test['distance2'] = np.sqrt(( df_test['pickup_pca0']-df_test['dropoff_pca0'])**2
+(df_test['pickup_pca1']-df_test['dropoff_pca1'])**2 )
pred = pd.DataFrame({"id": test['Id'], "SalePrice": pred})
 pred.to_csv('sample_submission.csv',index=False )<set_options>
df['log_trip_duration'] = np.log(df['trip_duration'] )
warnings.simplefilter(action="ignore", category=FutureWarning)
 pd.set_option("display.max_rows", 2000)
 pd.set_option("display.max_columns", 500)
 pd.set_option("display.width", 100)
 pd.set_option("display.max_colwidth", 500)
 pd.set_option("display.float_format", lambda x: "%.2f" % x)
 display(HTML("<style>.container { width:100% !important; }</style>"))
cwd = os.getcwd()
print(cwd)
 gc.enable()
BasicTicker,
ColorBar,
ColumnDataSource,
LinearColorMapper,
PrintfTickFormatter, 
)
 random_state = 55 
<set_options>
NUM_VARS = ['pickup_pca0', 'pickup_pca1', 'dropoff_pca0', 'dropoff_pca1', 'month', 'hour', 'day', 'distance2']
TARGET = 'log_trip_duration'
%%javascript
IPython.OutputArea.auto_scroll_threshold = 9999;<load_from_csv>
num_features = NUM_VARS
train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv" )<drop_column>
X_train = df.loc[:, num_features]
y_train = df[TARGET]
X_test = df_test.loc[:, num_features]
X_train.shape, y_train.shape, X_test.shape
Id = "Id"
submission_ID = test.loc[:, Id]
train.drop(Id, axis=1, inplace=True)
 test.drop(Id, axis=1, inplace=True)
 train.loc[:, "Train"] = 1
test.loc[:, "Train"] = 0
test["SalePrice"] = 0
stacked_DF = pd.concat([train, test], ignore_index=True)
<set_options>
from sklearn.ensemble import RandomForestRegressor
params = {"axes.labelsize": 20,
"xtick.labelsize": 14,
"ytick.labelsize": 14}
plt.rcParams.update(params)
 features_to_viz = [
"GrLivArea",
"YearBuilt",
"WoodDeckSF",
"LotArea",
"GarageArea",
"1stFlrSF",
"2ndFlrSF",
"TotalBsmtSF",
"LotFrontage",
"GarageYrBlt",
]
features_to_viz = sorted(features_to_viz)
 ncols = 1
nrows = math.ceil(len(features_to_viz)/ ncols)
 unused = nrows * ncols - len(features_to_viz)
 figw = ncols * 10
figh = nrows * 8 
(fig, ax)= plt.subplots(nrows, ncols, sharey=True, figsize=(figw, figh))
fig.subplots_adjust(hspace=0.3)
 ax = ax.flatten()
for i in range(unused, 0, -1):
fig.delaxes(ax[-i])
 for(n, col)in enumerate(features_to_viz):
if n % 2 != 0:
ax[n].yaxis.label.set_visible(False)
 ax[n].set_xlabel(col)
 ax[n].set_ylabel("SalePrice")
 sns.scatterplot(
 x=col,
y="SalePrice",
data=stacked_DF.loc[stacked_DF["Train"] == 1],
hue="SalePrice",
palette="gist_earth",
s=75,
legend=False,
ax=ax[n], 
)
 plt.show() 
<define_variables>
from sklearn.ensemble import RandomForestRegressor
features_to_viz = [
"Neighborhood",
"BsmtQual",
"ExterQual",
"FireplaceQu",
"ExterCond",
"KitchenQual",
"LotShape",
"OverallQual",
"FullBath",
"HalfBath",
"TotRmsAbvGrd",
"Fireplaces",
"KitchenAbvGr",
]
features_to_viz = sorted(features_to_viz)
 ncols = 1
nrows = math.ceil(len(features_to_viz)/ ncols)
 unused = nrows * ncols - len(features_to_viz)
(figw, figh)=(ncols * 10, nrows * 8)
(fig, ax)= plt.subplots(nrows, ncols, figsize=(figw, figh))
fig.subplots_adjust(hspace=0.2, wspace=0.2)
 for(n, col)in enumerate(features_to_viz):
ordering =(
 stacked_DF.loc[stacked_DF["Train"] == 1] 
.groupby(by=col)["SalePrice"] 
.median() 
.sort_values() 
.index 
)
 sns.boxplot(
 x="SalePrice",
y=col,
data=stacked_DF.loc[stacked_DF["Train"] == 1],
order=ordering,
ax=ax[n],
orient="h", 
)
 plt.show()<count_missing_values>
m = RandomForestRegressor(n_estimators=100, min_samples_leaf=5, min_samples_split=15, max_features='auto', bootstrap=True)
 m.fit(X_train, y_train )
print("Missing Value Counts in Train DF")
 stacked_DF[stacked_DF["Train"] == 1].isna().sum() [
stacked_DF[stacked_DF["Train"] == 1].isna().sum() > 0
].sort_values(ascending=False )<sort_values>
from sklearn.model_selection import cross_val_score
print("Missing Values in Test DF")
 stacked_DF[stacked_DF["Train"] == 0].isna().sum() [
stacked_DF[stacked_DF["Train"] == 0].isna().sum() > 0
].sort_values(ascending=False)
<categorify>
cv_scores = cross_val_score(m, X_train, y_train, cv=5, scoring='neg_mean_squared_log_error')
 cv_scores
lookup =(
 stacked_DF.loc[stacked_DF["Train"] == 1] 
.groupby(by="Neighborhood")["MSZoning"] 
.agg(pd.Series.mode)
)
 stacked_DF["MSZoning"] = stacked_DF["MSZoning"].fillna(
 stacked_DF["Neighborhood"].map(lookup)
)
 lookup =(
 stacked_DF.loc[stacked_DF["Train"] == 1] 
.groupby(by="OverallQual")["KitchenQual"] 
.agg(pd.Series.mode)
)
 stacked_DF["KitchenQual"] = stacked_DF["KitchenQual"].fillna(
 stacked_DF["OverallQual"].map(lookup)
)
 cols_na_to_missing = {
"Alley",
"BsmtCond",
"BsmtExposure",
"BsmtFinType1",
"BsmtFullBath",
"BsmtQual",
"Fence",
"FireplaceQu",
"GarageCond",
"GarageFinish",
"GarageQual",
"GarageType",
"MasVnrType",
"MiscFeature",
"PoolQC",
"BsmtFinType2",
}
cols_na_to_zero = {
"GarageArea",
"GarageCars",
"TotalBsmtSF",
"MasVnrArea",
"BsmtFinSF1",
"BsmtFinSF2",
"BsmtFullBath",
"BsmtHalfBath",
"GarageYrBlt",
}
cols_na_to_mode = {
"Functional",
"Electrical",
"Utilities",
"Exterior1st",
"Exterior2nd",
"SaleType",
}
for col in cols_na_to_missing:
stacked_DF[col] = stacked_DF[col].astype(object ).fillna("Missing")
 for col in cols_na_to_zero:
stacked_DF[col] = stacked_DF[col].astype(object ).fillna(0)
 for col in cols_na_to_mode:
stacked_DF[col] =(
 stacked_DF[col] 
.astype(object)
.fillna(stacked_DF.loc[stacked_DF["Train"] == 1, col].mode() [0])
)
 num_features = stacked_DF.drop(columns=["Train"] ).select_dtypes("number" ).columns
imputer = IterativeImputer(
 RandomForestRegressor(max_depth=8),
n_nearest_features=10,
max_iter=10,
random_state=random_state, 
)
 stacked_DF.loc[stacked_DF["Train"] == 1, num_features] = imputer.fit_transform(
 stacked_DF.loc[stacked_DF["Train"] == 1, num_features].values 
)
 stacked_DF.loc[stacked_DF["Train"] == 0, num_features] = imputer.transform(
 stacked_DF.loc[stacked_DF["Train"] == 0, num_features].values 
)
<feature_engineering>
y_test_pred = m.predict(X_test)
 y_test_pred[:5]
<feature_engineering><EOS> 
my_submission = pd.DataFrame({'id': df_test.id, 'trip_duration': np.exp(y_test_pred)})
 my_submission.to_csv('submission.csv', index=False )