Kaggle competition: New York City Taxi Trip Duration
<SOS> metric: RMSLE Kaggle data source: new-york-city-taxi-trip-duration<prepare_x_and_y>
warnings.filterwarnings('ignore')
 pd.set_option('display.max_colwidth', -1)
 plt.style.use('fivethirtyeight')
X_train = train_df.drop(["ID", "target"], axis=1)
 y_train = np.log1p(train_df["target"].values)
 X_test = test_df.drop(["ID"], axis=1 )<drop_column>
train=pd.read_csv(".. /input/train.csv")
 test=pd.read_csv(".. /input/test.csv")
 test.shape
colsToRemove = []
for col in X_train.columns:
if X_train[col].std() == 0:
colsToRemove.append(col)
 X_train.drop(colsToRemove, axis=1, inplace=True)
 X_test.drop(colsToRemove, axis=1, inplace=True)
 print("Removed `{}` Constant Columns
".format(len(colsToRemove)))
 print(colsToRemove )<define_variables>
train['pickup_datetime']=pd.to_datetime(train['pickup_datetime'],format='%Y-%m-%d %H:%M:%S')
 train['dropoff_datetime']=pd.to_datetime(train['dropoff_datetime'],format='%Y-%m-%d %H:%M:%S')
 train.head()
%%time
colsToRemove = []
colsScaned = []
dupList = {}
columns = X_train.columns
for i in range(len(columns)-1):
v = X_train[columns[i]].values
dupCols = []
for j in range(i+1,len(columns)) :
if np.array_equal(v, X_train[columns[j]].values):
colsToRemove.append(columns[j])
 if columns[j] not in colsScaned:
dupCols.append(columns[j])
 colsScaned.append(columns[j])
 dupList[columns[i]] = dupCols
X_train.drop(colsToRemove, axis=1, inplace=True)
 X_test.drop(colsToRemove, axis=1, inplace=True)
 print("Removed `{}` Duplicate Columns
".format(len(dupList)))
 print(dupList )<drop_column>
train['pickup_date']= train['pickup_datetime'].dt.date
train['pickup_day']=train['pickup_datetime'].apply(lambda x:x.day)
 train['pickup_hour']=train['pickup_datetime'].apply(lambda x:x.hour)
 train['pickup_day_of_week']=train['pickup_datetime'].apply(lambda x:calendar.day_name[x.weekday() ])
 train['dropoff_date']= train['dropoff_datetime'].dt.date
train['dropoff_day']=train['dropoff_datetime'].apply(lambda x:x.day)
 train['dropoff_hour']=train['dropoff_datetime'].apply(lambda x:x.hour)
 train['dropoff_day_of_week']=train['dropoff_datetime'].apply(lambda x:calendar.day_name[x.weekday() ] )
def drop_sparse(train, test):
flist = [x for x in train.columns if not x in ['ID','target']]
for f in flist:
if len(np.unique(train[f])) <2:
train.drop(f, axis=1, inplace=True)
 test.drop(f, axis=1, inplace=True)
 return train, test<drop_column>
train['pickup_latitude_round3']=train['pickup_latitude'].apply(lambda x:round(x,3))
train['pickup_longitude_round3']=train['pickup_longitude'].apply(lambda x:round(x,3))
train['dropoff_latitude_round3']=train['dropoff_latitude'].apply(lambda x:round(x,3))
train['dropoff_longitude_round3']=train['dropoff_longitude'].apply(lambda x:round(x,3))
train.head()
%%time
X_train, X_test = drop_sparse(X_train, X_test )<data_type_conversions>
def calculateDistance(row):
R=6373.0
pickup_lat=radians(row['pickup_latitude'])
 pickup_lon=radians(row['pickup_longitude'])
 dropoff_lat=radians(row['dropoff_latitude'])
 dropoff_lon=radians(row['dropoff_longitude'])
 dlon = dropoff_lon - pickup_lon
dlat = dropoff_lat - pickup_lat
a = sin(dlat / 2)**2 + cos(pickup_lat)* cos(dropoff_lat)* sin(dlon / 2)**2
c = 2 * atan2(sqrt(a), sqrt(1 - a))
distance = R * c
return distance
def add_SumZeros(train, test, features):
flist = [x for x in train.columns if not x in ['ID','target']]
if 'SumZeros' in features:
train.insert(1, 'SumZeros',(train[flist] == 0 ).astype(int ).sum(axis=1))
test.insert(1, 'SumZeros',(test[flist] == 0 ).astype(int ).sum(axis=1))
flist = [x for x in train.columns if not x in ['ID','target']]
return train, test<compute_test_metric>
train['trip_distance']=train.apply(lambda row:calculateDistance(row),axis=1)
 train.head()
%%time
X_train, X_test = add_SumZeros(X_train, X_test, ['SumZeros'] )<data_type_conversions>
train['trip_duration_in_hour']=train['trip_duration'].apply(lambda x:x/3600)
 train.head()
def add_SumValues(train, test, features):
flist = [x for x in train.columns if not x in ['ID','target']]
if 'SumValues' in features:
train.insert(1, 'SumValues',(train[flist] != 0 ).astype(int ).sum(axis=1))
test.insert(1, 'SumValues',(test[flist] != 0 ).astype(int ).sum(axis=1))
flist = [x for x in train.columns if not x in ['ID','target']]
return train, test<compute_test_metric>
outlier_trip_duration=train.loc[train['trip_duration_in_hour']>24]
outlier_trip_duration
%%time
X_train, X_test = add_SumValues(X_train, X_test, ['SumValues'] )<feature_engineering>
outlier_trip_duration=train.loc[train['trip_duration_in_hour']>24]
outlier_trip_duration
def add_OtherAgg(train, test, features):
flist = [x for x in train.columns if not x in ['ID','target','SumZeros','SumValues']]
if 'OtherAgg' in features:
train['Mean'] = train[flist].mean(axis=1)
 train['Median'] = train[flist].median(axis=1)
 train['Mode'] = train[flist].mode(axis=1)
 train['Max'] = train[flist].max(axis=1)
 train['Var'] = train[flist].var(axis=1)
 train['Std'] = train[flist].std(axis=1)
 test['Mean'] = test[flist].mean(axis=1)
 test['Median'] = test[flist].median(axis=1)
 test['Mode'] = test[flist].mode(axis=1)
 test['Max'] = test[flist].max(axis=1)
 test['Var'] = test[flist].var(axis=1)
 test['Std'] = test[flist].std(axis=1)
 flist = [x for x in train.columns if not x in ['ID','target','SumZeros','SumValues']]
return train, test<concatenate>
pickup=train.groupby(['pickup_latitude_round3','pickup_longitude_round3'])['id'].count().reset_index().rename(columns={'id':'Num_Trips'} )
%%time
X_train, X_test = add_OtherAgg(X_train, X_test, ['OtherAgg'] )<import_modules>
drop=train.groupby(['dropoff_latitude_round3','dropoff_longitude_round3'])['id'].count().reset_index().rename(columns={'id':'Num_Trips'} )
from sklearn.random_projection import GaussianRandomProjection, SparseRandomProjection
from sklearn.decomposition import TruncatedSVD, FastICA<categorify>
pickup=train.groupby(['pickup_latitude_round3','pickup_longitude_round3'])['trip_duration'].mean().reset_index().rename(columns={'trip_duration':'Avg_Trip_duration'} )
PERC_TRESHOLD = 0.98
N_COMP = 97
print("tSVD")
 tsvd = TruncatedSVD(n_components=N_COMP, random_state=17)
 tsvd_results_train = tsvd.fit_transform(X_train)
 tsvd_results_test = tsvd.transform(X_test)
 print("Append decomposition components to datasets...")
 for i in range(1, N_COMP + 1):
X_train['tsvd_' + str(i)] = tsvd_results_train[:, i - 1]
X_test['tsvd_' + str(i)] = tsvd_results_test[:, i - 1]
print('
Train shape: {}
Test shape: {}'.format(X_train.shape, X_test.shape))<import_modules>
avg_duration_day=train.groupby(['pickup_day_of_week'])['trip_duration'].mean().reset_index().rename(columns={'trip_duration':'avg_trip_duration'} )
import lightgbm as lgb<train_model>
def calculateBearing(lat1,lng1,lat2,lng2):
R = 6371
lng_delta_rad = np.radians(lng2 - lng1)
 lat1, lng1, lat2, lng2 = map(np.radians,(lat1, lng1, lat2, lng2))
y = np.sin(lng_delta_rad)* np.cos(lat2)
 x = np.cos(lat1)* np.sin(lat2)- np.sin(lat1)* np.cos(lat2)* np.cos(lng_delta_rad)
 return np.degrees(np.arctan2(y, x))
def run_lgb(train_X, train_y, val_X, val_y, test_X):
params = {
"objective" : "regression",
"metric" : "rmse",
'boosting_type' : 'goss',
'max_depth' : 5,
"num_leaves" : 20,
"learning_rate" : 0.01,
"feature_fraction" : 0.6,
"bagging_seed" : 42,
"verbosity" : -1,
'lambda_l2' : 0.000001,
'lambda_l1' : 0.00001,
'max_bin' : 200
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 evals_result = {}
model = lgb.train(params, lgtrain, 1000, valid_sets=[lgtrain, lgval], early_stopping_rounds=100,
verbose_eval=200, evals_result=evals_result)
 pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
 return pred_test_y, model, evals_result<split>
train['bearing']=train.apply(lambda row:calculateBearing(row['pickup_latitude_round3'],row['pickup_longitude_round3'],row['dropoff_latitude_round3'],row['dropoff_longitude_round3']),axis=1 )
seeds = [42]
pred_test_full_seed = 0
for seed in seeds:
kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=seed)
 pred_test_full = 0
for dev_index, val_index in kf.split(X_train):
dev_X, val_X = X_train.loc[dev_index,:], X_train.loc[val_index,:]
dev_y, val_y = y_train[dev_index], y_train[val_index]
pred_test, model, evals_result = run_lgb(dev_X, dev_y, val_X, val_y, X_test)
 pred_test_full += pred_test
pred_test_full /= 5.
 pred_test_full = np.expm1(pred_test_full)
 pred_test_full_seed += pred_test_full
print("Seed {} completed.... ".format(seed))
pred_test_full_seed /= np.float(len(seeds))
print("LightGBM Training Completed..." )<load_from_csv>
train['store_and_fwd_flag'].value_counts()
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub_lgb = pd.DataFrame()
sub_lgb["target"] = pred_test_full_seed
sub["target"] = sub_lgb["target"]<save_to_csv>
coords = np.vstack(( train[['pickup_latitude', 'pickup_longitude']].values,
train[['dropoff_latitude', 'dropoff_longitude']].values,
test[['pickup_latitude', 'pickup_longitude']].values,
test[['dropoff_latitude', 'dropoff_longitude']].values))
kmeans = KMeans(n_clusters=8, random_state=0 ).fit(coords)
 train.loc[:, 'pickup_neighbourhood'] = kmeans.predict(train[['pickup_latitude', 'pickup_longitude']])
 train.loc[:, 'dropoff_neighbourhood'] = kmeans.predict(train[['dropoff_latitude', 'dropoff_longitude']])
 city_long_border =(-74.03, -73.75)
 city_lat_border =(40.63, 40.85)
 fig, ax = plt.subplots(ncols=1, sharex=True, sharey=True)
 ax.scatter(train['pickup_longitude'], train['pickup_latitude'],
c=train['pickup_neighbourhood'], label='train', alpha=0.1)
 fig.suptitle('Pickup Neighbourhood')
 ax.set_ylabel('latitude')
 ax.set_xlabel('longitude')
 plt.ylim(city_lat_border)
 plt.xlim(city_long_border )
print(sub.head())
 sub.to_csv('Ensemble9.csv', index=False )<train_model>
print("Average speed is",np.mean(train['avg_speed_kph']),"kph" )
class SklearnWrapper(object):
def __init__(self, clf, seed=0, params=None, seed_bool = True):
if(seed_bool == True):
params['random_state'] = seed
self.clf = clf(**params)
 def train(self, x_train, y_train):
self.clf.fit(x_train, y_train)
 def predict(self, x):
return self.clf.predict(x)
 def get_oof(clf, x_train, y, x_test):
oof_train = np.zeros(( ntrain,))
oof_test = np.zeros(( ntest,))
oof_test_skf = np.empty(( NFOLDS, ntest))
for i,(train_index, test_index)in enumerate(kf):
print('
Fold {}'.format(i))
x_tr = x_train.iloc[train_index]
y_tr = y[train_index]
x_te = x_train.iloc[test_index]
clf.train(x_tr, y_tr)
 oof_train[test_index] = clf.predict(x_te)
 oof_test_skf[i, :] = clf.predict(x_test)
 oof_test[:] = oof_test_skf.mean(axis=0)
 return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)
<load_from_csv>
test['pickup_datetime']=pd.to_datetime(test['pickup_datetime'],format='%Y-%m-%d %H:%M:%S')
 test['pickup_date']= test['pickup_datetime'].dt.date
test['pickup_day']=test['pickup_datetime'].apply(lambda x:x.day)
 test['pickup_hour']=test['pickup_datetime'].apply(lambda x:x.hour)
 test['pickup_day_of_week']=test['pickup_datetime'].apply(lambda x:calendar.day_name[x.weekday() ])
 test['pickup_latitude_round3']=test['pickup_latitude'].apply(lambda x:round(x,3))
test['pickup_longitude_round3']=test['pickup_longitude'].apply(lambda x:round(x,3))
test['dropoff_latitude_round3']=test['dropoff_latitude'].apply(lambda x:round(x,3))
test['dropoff_longitude_round3']=test['dropoff_longitude'].apply(lambda x:round(x,3))
test['trip_distance']=test.apply(lambda row:calculateDistance(row),axis=1)
 test['bearing']=test.apply(lambda row:calculateBearing(row['pickup_latitude_round3'],row['pickup_longitude_round3'],row['dropoff_latitude_round3'],row['dropoff_longitude_round3']),axis=1)
 test.loc[:, 'pickup_neighbourhood'] = kmeans.predict(test[['pickup_latitude', 'pickup_longitude']])
 test.loc[:, 'dropoff_neighbourhood'] = kmeans.predict(test[['dropoff_latitude', 'dropoff_longitude']] )
train_df = pd.read_csv('.. /input/santander-value-prediction-challenge/train.csv')
 test_df = pd.read_csv('.. /input/santander-value-prediction-challenge/test.csv' )<prepare_x_and_y>
drop_cols=['avg_speed_kph','trip_duration_in_hour','dropoff_date','dropoff_day','dropoff_hour','dropoff_day_of_week','dropoff_datetime','pickup_latitude','pickup_longitude','dropoff_latitude','dropoff_longitude']
training=train.drop(drop_cols,axis=1)
 testing=test.drop(['pickup_latitude','pickup_longitude','dropoff_latitude','dropoff_longitude'],axis=1)
X_train = train_df.drop(["ID", "target"], axis=1)
 y_train = np.log1p(train_df["target"].values)
 X_test = test_df.drop(["ID"], axis=1)
 ntrain = len(X_train)
 ntest = len(X_test )<drop_column>
training['log_trip_duration']=training['trip_duration'].apply(lambda x:np.log(x))
training.drop(['trip_duration'],axis=1,inplace=True )
colsToRemove = []
for col in X_train.columns:
if X_train[col].std() == 0:
colsToRemove.append(col)
 X_train.drop(colsToRemove, axis=1, inplace=True)
 X_test.drop(colsToRemove, axis=1, inplace=True)
<define_variables>
def encodeDays(day_of_week):
day_dict={'Sunday':0,'Monday':1,'Tuesday':2,'Wednesday':3,'Thursday':4,'Friday':5,'Saturday':6}
return day_dict[day_of_week]
colsToRemove = []
colsScaned = []
dupList = {}
columns = X_train.columns
for i in range(len(columns)-1):
v = X_train[columns[i]].values
dupCols = []
for j in range(i+1,len(columns)) :
if np.array_equal(v, X_train[columns[j]].values):
colsToRemove.append(columns[j])
 if columns[j] not in colsScaned:
dupCols.append(columns[j])
 colsScaned.append(columns[j])
 dupList[columns[i]] = dupCols
X_train.drop(colsToRemove, axis=1, inplace=True)
 X_test.drop(colsToRemove, axis=1, inplace=True )<feature_engineering>
training['pickup_day_of_week']=training['pickup_day_of_week'].apply(lambda x:encodeDays(x))
testing['pickup_day_of_week']=testing['pickup_day_of_week'].apply(lambda x:encodeDays(x))
weight =(( X_train != 0 ).sum() /len(X_train)).values
tmp_train = X_train[X_train!=0].fillna(0)
 tmp_test = X_test[X_test!=0].fillna(0)
 X_train["weight_count"] =(tmp_train*weight ).sum(axis=1)
 X_test["weight_count"] =(tmp_test*weight ).sum(axis=1)
 X_train["count_not0"] =(X_train != 0 ).sum(axis=1)
 X_test["count_not0"] =(X_test != 0 ).sum(axis=1)
 X_train["sum"] = X_train.sum(axis=1)
 X_test["sum"] = X_test.sum(axis=1)
 X_train["var"] = X_train.var(axis=1)
 X_test["var"] = X_test.var(axis=1)
 X_train["mean"] = X_train.mean(axis=1)
 X_test["mean"] = X_test.mean(axis=1)
 X_train["std"] = X_train.std(axis=1)
 X_test["std"] = X_test.std(axis=1)
 X_train["max"] = X_train.max(axis=1)
 X_test["max"] = X_test.max(axis=1)
 X_train["min"] = X_train.min(axis=1)
 X_test["min"] = X_test.min(axis=1)
<choose_model_class>
training.to_csv("input_training.csv",index=False)
 testing.to_csv("input_testing.csv",index=False)
 del training
del testing
del train
del test
SEED = 5
NFOLDS = 5
kf = KFold(ntrain, n_folds=NFOLDS, shuffle=True, random_state=SEED)
 ridge_params = {'alpha':30.0, 'fit_intercept':True, 'normalize':False, 'copy_X':True,
'max_iter':None, 'tol':0.001, 'solver':'auto', 'random_state':SEED}
ridge = SklearnWrapper(clf=Ridge, seed = SEED, params = ridge_params)
 ridge_oof_train, ridge_oof_test = get_oof(ridge, X_train, y_train, X_test)
<compute_test_metric>
def LabelEncoding(train_df,test_df,max_levels=2):
for col in train_df:
if train_df[col].dtype == 'object':
if len(list(train_df[col].unique())) <= max_levels:
le = preprocessing.LabelEncoder()
le.fit(train_df[col])
 train_df[col]=le.transform(train_df[col])
 test_df[col]=le.transform(test_df[col])
 return [train_df,test_df]
def readInputAndEncode(input_path,train_file,test_file,target_column):
training=pd.read_csv(input_path+train_file)
 testing=pd.read_csv(input_path+test_file)
 training,testing=LabelEncoding(training,testing)
 train_cols=training.columns.tolist()
test_cols=testing.columns.tolist()
col_in_train_not_test=set(train_cols)-set(test_cols)
 for col in col_in_train_not_test:
if col!=target_column:
testing[col]=0
col_in_test_not_train=set(test_cols)-set(train_cols)
 for col in col_in_test_not_train:
training[col]=0
print("Training Data Shape after Processing ",training.shape)
 print("Testing Data Shape after Processing ",testing.shape)
 return [training,testing]
rms = sqrt(mean_squared_error(y_train, ridge_oof_train))
print('Ridge OOF RMSE: {}'.format(rms))
print("Modeling Stage")
 X_train["ridge_preds"] = ridge_oof_train
X_test["rigde_preds"] = ridge_oof_test<split>
train,test=readInputAndEncode("",'input_training.csv','input_testing.csv','log_trip_duration')
 train.drop(['pickup_date'],axis=1,inplace=True)
 test.drop(['pickup_date'],axis=1,inplace=True)
 train.drop(['pickup_datetime'],axis=1,inplace=True)
 test.drop(['pickup_datetime'],axis=1,inplace=True)
 test_id=test['id']
train.drop(['id'],axis=1,inplace=True)
 test.drop(['id'],axis=1,inplace=True )
dev_X, val_X, dev_y, val_y = train_test_split(X_train, y_train, test_size = 0.2, random_state = 42 )<create_dataframe>
def GetFeaturesAndSplit(train,test,target,imputing_strategy='median',split=0.25,imputation=True):
labels=np.array(train[target])
 training=train.drop(target, axis = 1)
 training = np.array(training)
 testing=np.array(test)
 if imputation==True:
imputer=Imputer(strategy=imputing_strategy,missing_values=np.nan)
 imputer.fit(training)
 training=imputer.transform(training)
 testing=imputer.transform(testing)
 train_features, validation_features, train_labels, validation_labels = train_test_split(training, labels, test_size = split, random_state = 42)
 return [train_features,validation_features,train_labels,validation_labels,testing]
def run_lgb(train_X, train_y, val_X, val_y, test_X):
params = {
"objective" : "regression",
"metric" : "rmse",
"num_leaves" : 40,
"learning_rate" : 0.005,
"bagging_fraction" : 0.7,
"feature_fraction" : 0.5,
"bagging_frequency" : 5,
"bagging_seed" : 42,
"verbosity" : -1,
"seed": 42
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 evals_result = {}
model = lgb.train(params, lgtrain, 5000,
valid_sets=[lgval],
early_stopping_rounds=100,
verbose_eval=50,
evals_result=evals_result)
 pred_test_y = np.expm1(model.predict(test_X, num_iteration=model.best_iteration))
return pred_test_y, model, evals_result<train_model>
train_features,validation_features,train_labels,validation_labels,testing=GetFeaturesAndSplit(train,test,'log_trip_duration',imputation=False )
pred_test, model, evals_result = run_lgb(dev_X, dev_y, val_X, val_y, X_test)
 print("LightGBM Training Completed..." )<load_from_csv>
lm=linear_model.LinearRegression()
lm.fit(train_features,train_labels )
sub = pd.read_csv('.. /input/santander-value-prediction-challenge/sample_submission.csv')
 sub["target"] = pred_test
print(sub.head())
<save_to_csv>
valid_pred=lm.predict(validation_features )
sub.to_csv('sub_lgbm_ridge_agg.csv', index=False )<load_from_csv>
rmse=mean_squared_error(validation_labels,valid_pred)
 print("Root Mean Squared Error for Linear Regression(log scale)",rmse )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 test_id = test.ID
print(train.shape , "Train shape")
 print(test.shape , "Test shape" )<drop_column>
test_pred=lm.predict(testing)
 submit=pd.DataFrame()
submit['id']=test_id
submit['trip_duration']=np.exp(test_pred)
 submit.to_csv("submission_linear_regression_baseline.csv",index=False)
 del submit
Y = train.target
Y = np.log(Y+1)
 train = train.drop(['ID','target'], axis = 1)
 test = test.drop(['ID'], axis = 1 )<count_unique_values>
rf = RandomForestRegressor(n_estimators = 100, random_state = 42 )
unique_df = train.nunique().reset_index()
unique_df.columns = ["col_name", "unique_count"]
constant_df = unique_df[unique_df["unique_count"]==1]
constant_df.shape<drop_column>
rf.fit(train_features,train_labels )
train = train.drop(constant_df["col_name"], axis=1)
 test = test.drop(constant_df["col_name"], axis=1 )<sort_values>
valid_pred_rf=rf.predict(validation_features)
 rmse=mean_squared_error(validation_labels,valid_pred_rf)
 print("Root Mean Squared Error for Random Forest",rmse)
total =(train == 0 ).sum().sort_values(ascending = False)
 percent =(( train == 0 ).sum() /(train==0 ).count() *100 ).sort_values(ascending = False)
 train_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 train_data.tail(60 )<drop_column>
test_pred=rf.predict(testing)
 submit=pd.DataFrame()
submit['id']=test_id
submit['trip_duration']=np.exp(test_pred)
 submit.to_csv("submission_random_forest_baseline.csv",index=False )