Kaggle competition: New York City Taxi Trip Duration
<SOS> metric: RMSLE Kaggle data source: new-york-city-taxi-trip-duration<categorify>
%matplotlib inline
def convert_OHE2LE(df):
tmp_df = df.copy(deep=True)
 for s_ in ['pared', 'piso', 'techo', 'abastagua', 'sanitario', 'energcocinar', 'elimbasu',
'epared', 'etecho', 'eviv', 'estadocivil', 'parentesco',
'instlevel', 'lugar', 'tipovivi',
'manual_elec']:
if 'manual_' not in s_:
cols_s_ = [f_ for f_ in df.columns if f_.startswith(s_)]
elif 'elec' in s_:
cols_s_ = ['public', 'planpri', 'noelec', 'coopele']
sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print('The OHE in {} is incomplete.A new column will be added before label encoding' 
.format(s_))
col_dummy = s_+'_dummy'
tmp_df[col_dummy] =(tmp_df[cols_s_].sum(axis=1)== 0 ).astype(np.int8)
 cols_s_.append(col_dummy)
 sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print("The category completion did not work")
 tmp_cat = tmp_df[cols_s_].idxmax(axis=1)
 tmp_df[s_ + '_LE'] = LabelEncoder().fit_transform(tmp_cat ).astype(np.int16)
 if 'parentesco1' in cols_s_:
cols_s_.remove('parentesco1')
 tmp_df.drop(cols_s_, axis=1, inplace=True)
 return tmp_df<load_from_csv>
train=pd.read_csv(".. /input/taxi-trip-duration-train-csv/train.csv")
 train.head(3 )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 test_ids = test.Id<feature_engineering>
test=pd.read_csv(".. /input/new-york-taxi-trip-duration-test-csv/test.csv")
 test.head(3 )
train['dependency'] = np.sqrt(train['SQBdependency'])
 test['dependency'] = np.sqrt(test['SQBdependency'] )<categorify>
train.isnull().any()
def process_df(df_):
encode_data(df_)
 return do_features(df_)
 train = process_df(train)
 test = process_df(test )<data_type_conversions>
test.isnull().any()
train.loc[train['edjefa'] == "no", "edjefa"] = 0
train.loc[train['edjefa'] == "yes", "edjefa"] = 1
train['edjefa'] = train['edjefa'].astype("int")
 train.loc[train['edjefe'] == "no", "edjefe"] = 0
train.loc[train['edjefe'] == "yes", "edjefe"] = 1
train['edjefe'] = train['edjefe'].astype("int")
 test.loc[test['edjefa'] == "no", "edjefa"] = 0
test.loc[test['edjefa'] == "yes", "edjefa"] = 4
test['edjefa'] = test['edjefa'].astype("int")
 test.loc[test['edjefe'] == "no", "edjefe"] = 0
test.loc[test['edjefe'] == "yes", "edjefe"] = 4
test['edjefe'] = test['edjefe'].astype("int")
 train['edjef'] = np.max(train[['edjefa','edjefe']], axis=1)
 test['edjef'] = np.max(test[['edjefa','edjefe']], axis=1)
 train['v2a1']=train['v2a1'].fillna(0)
 test['v2a1']=test['v2a1'].fillna(0)
 test['v18q1']=test['v18q1'].fillna(0)
 train['v18q1']=train['v18q1'].fillna(0)
 train['rez_esc']=train['rez_esc'].fillna(0)
 test['rez_esc']=test['rez_esc'].fillna(0)
 train.loc[train.meaneduc.isnull() , "meaneduc"] = 0
train.loc[train.SQBmeaned.isnull() , "SQBmeaned"] = 0
test.loc[test.meaneduc.isnull() , "meaneduc"] = 0
test.loc[test.SQBmeaned.isnull() , "SQBmeaned"] = 0
train.loc[(train.v14a == 1)&(train.sanitario1 == 1)&(train.abastaguano == 0), "v14a"] = 0
train.loc[(train.v14a == 1)&(train.sanitario1 == 1)&(train.abastaguano == 0), "sanitario1"] = 0
test.loc[(test.v14a == 1)&(test.sanitario1 == 1)&(test.abastaguano == 0), "v14a"] = 0
test.loc[(test.v14a == 1)&(test.sanitario1 == 1)&(test.abastaguano == 0), "sanitario1"] = 0<concatenate>
m = np.mean(train['trip_duration'])
 s = np.std(train['trip_duration'])
 train = train[train['trip_duration'] <= m + 2*s]
train = train[train['trip_duration'] >= m - 2*s]
def train_test_apply_func(train_, test_, func_):
test_['Target'] = 0
xx = pd.concat([train_, test_])
 xx_func = func_(xx)
 train_ = xx_func.iloc[:train_.shape[0], :]
test_ = xx_func.iloc[train_.shape[0]:, :].drop('Target', axis=1)
 del xx, xx_func
return train_, test_<feature_engineering>
west, south, east, north = -74.03, 40.63, -73.77, 40.85
train = train[(train.pickup_latitude> south)&(train.pickup_latitude < north)]
train = train[(train.dropoff_latitude> south)&(train.dropoff_latitude < north)]
train = train[(train.pickup_longitude> west)&(train.pickup_longitude < east)]
train = train[(train.dropoff_longitude> west)&(train.dropoff_longitude < east)]
train, test = train_test_apply_func(train, test, convert_OHE2LE )<categorify>
train['pickup_datetime'] = pd.to_datetime(train.pickup_datetime)
 test['pickup_datetime'] = pd.to_datetime(test.pickup_datetime)
 train['pickup_date'] = train['pickup_datetime'].dt.date
test['pickup_date'] = test['pickup_datetime'].dt.date
train['Month'] = train['pickup_datetime'].dt.month
test['Month'] = test['pickup_datetime'].dt.month
train['dayofweek'] = train['pickup_datetime'].dt.dayofweek
test['dayofweek'] = test['pickup_datetime'].dt.dayofweek
train['DayofMonth'] = train['pickup_datetime'].dt.day
test['DayofMonth'] = test['pickup_datetime'].dt.day
train['Hour'] = train['pickup_datetime'].dt.hour
test['Hour'] = test['pickup_datetime'].dt.hour
train['dropoff_datetime'] = pd.to_datetime(train.dropoff_datetime )
cols_2_ohe = ['eviv_LE', 'etecho_LE', 'epared_LE', 'elimbasu_LE',
'energcocinar_LE', 'sanitario_LE', 'manual_elec_LE',
'pared_LE']
cols_nums = ['age', 'meaneduc', 'dependency',
'hogar_nin', 'hogar_adul', 'hogar_mayor', 'hogar_total',
'bedrooms', 'overcrowding']
def convert_geo2aggs(df_):
tmp_df = pd.concat([df_[(['lugar_LE', 'idhogar']+cols_nums)],
pd.get_dummies(df_[cols_2_ohe],
columns=cols_2_ohe)],axis=1)
 geo_agg = tmp_df.groupby(['lugar_LE','idhogar'] ).mean().groupby('lugar_LE' ).mean().astype(np.float32)
 geo_agg.columns = pd.Index(['geo_' + e for e in geo_agg.columns.tolist() ])
 del tmp_df
return df_.join(geo_agg, how='left', on='lugar_LE')
 train, test = train_test_apply_func(train, test, convert_geo2aggs )<categorify>
month_usage = pd.value_counts(train['Month'] ).sort_index()
dow_usage = pd.value_counts(train['dayofweek'] ).sort_index()
hour_usage = pd.value_counts(train['Hour'] ).sort_index()
month_vendor = train.groupby(['Month', 'vendor_id'] ).size().unstack()
dow_vendor = train.groupby(['dayofweek', 'vendor_id'] ).size().unstack()
hour_vendor = train.groupby(['Hour', 'vendor_id'] ).size().unstack()
server = train.groupby(['store_and_fwd_flag', 'vendor_id'] ).size().unstack()
train['num_over_18'] = 0
train['num_over_18'] = train[train.age >= 18].groupby('idhogar' ).transform("count")
 train['num_over_18'] = train.groupby("idhogar")["num_over_18"].transform("max")
 train['num_over_18'] = train['num_over_18'].fillna(0)
 test['num_over_18'] = 0
test['num_over_18'] = test[test.age >= 18].groupby('idhogar' ).transform("count")
 test['num_over_18'] = test.groupby("idhogar")["num_over_18"].transform("max")
 test['num_over_18'] = test['num_over_18'].fillna(0)
 def extract_features(df):
df['bedrooms_to_rooms'] = df['bedrooms']/df['rooms']
df['rent_to_rooms'] = df['v2a1']/df['rooms']
df['tamhog_to_rooms'] = df['tamhog']/df['rooms']
df['r4t3_to_tamhog'] = df['r4t3']/df['tamhog']
df['r4t3_to_rooms'] = df['r4t3']/df['rooms']
df['v2a1_to_r4t3'] = df['v2a1']/df['r4t3']
df['v2a1_to_r4t3'] = df['v2a1']/(df['r4t3'] - df['r4t1'])
 df['hhsize_to_rooms'] = df['hhsize']/df['rooms']
df['rent_to_hhsize'] = df['v2a1']/df['hhsize']
df['rent_to_over_18'] = df['v2a1']/df['num_over_18']
df.loc[df.num_over_18 == 0, "rent_to_over_18"] = df[df.num_over_18 == 0].v2a1
df['dependency_yes'] = df['dependency'].apply(lambda x: 1 if x == 'yes' else 0)
 df['dependency_no'] = df['dependency'].apply(lambda x: 1 if x == 'no' else 0)
 extract_features(train)
 extract_features(test )<prepare_x_and_y>
def haversine_array(lat1, lng1, lat2, lng2):
lat1, lng1, lat2, lng2 = map(np.radians,(lat1, lng1, lat2, lng2))
AVG_EARTH_RADIUS = 6371
lat = lat2 - lat1
lng = lng2 - lng1
d = np.sin(lat * 0.5)** 2 + np.cos(lat1)* np.cos(lat2)* np.sin(lng * 0.5)** 2
h = 2 * AVG_EARTH_RADIUS * np.arcsin(np.sqrt(d))
return h
def dummy_manhattan_distance(lat1, lng1, lat2, lng2):
a = haversine_array(lat1, lng1, lat1, lng2)
 b = haversine_array(lat1, lng1, lat2, lng1)
 return a + b
def bearing_array(lat1, lng1, lat2, lng2):
AVG_EARTH_RADIUS = 6371
lng_delta_rad = np.radians(lng2 - lng1)
 lat1, lng1, lat2, lng2 = map(np.radians,(lat1, lng1, lat2, lng2))
y = np.sin(lng_delta_rad)* np.cos(lat2)
 x = np.cos(lat1)* np.sin(lat2)- np.sin(lat1)* np.cos(lat2)* np.cos(lng_delta_rad)
 return np.degrees(np.arctan2(y, x))
X = train.query('parentesco1==1')
 y = X['Target'] - 1
X = X.drop(['Target'], axis=1)
 np.random.seed(seed=None)
 train2 = X.copy()
train_hhs = train2.idhogar
households = train2.idhogar.unique()
cv_hhs = np.random.choice(households, size=int(len(households)* 0.1), replace=False)
 cv_idx = np.isin(train2.idhogar, cv_hhs)
 X_test = train2[cv_idx]
y_test = y[cv_idx]
X_train = train2[~cv_idx]
y_train = y[~cv_idx] 
<define_variables>
train['distance_haversine'] = haversine_array(train['pickup_latitude'].values, train['pickup_longitude'].values, train['dropoff_latitude'].values, train['dropoff_longitude'].values)
 test['distance_haversine'] = haversine_array(test['pickup_latitude'].values, test['pickup_longitude'].values, test['dropoff_latitude'].values, test['dropoff_longitude'].values)
 train['distance_dummy_manhattan'] = dummy_manhattan_distance(train['pickup_latitude'].values, train['pickup_longitude'].values, train['dropoff_latitude'].values, train['dropoff_longitude'].values)
 test['distance_dummy_manhattan'] = dummy_manhattan_distance(test['pickup_latitude'].values, test['pickup_longitude'].values, test['dropoff_latitude'].values, test['dropoff_longitude'].values)
 train['direction'] = bearing_array(train['pickup_latitude'].values, train['pickup_longitude'].values, train['dropoff_latitude'].values, train['dropoff_longitude'].values)
 test['direction'] = bearing_array(test['pickup_latitude'].values, test['pickup_longitude'].values, test['dropoff_latitude'].values, test['dropoff_longitude'].values )
extra_drop_features = ['mobilephone',
'hogar_total',
'agesq',
'hacapo',
'tamhog',
'agg18_estadocivil3_COUNT',
'agg18_estadocivil4_COUNT',
'v14a',
'agg18_estadocivil1_MEAN',
'fe_people_weird_stat',
'num_over_18',
'r4t3_to_tamhog',
'agg18_parentesco4_COUNT',
'agg18_estadocivil5_COUNT',
'agg18_parentesco7_COUNT',
'agg18_parentesco2_COUNT',
'agg18_parentesco5_COUNT',
'agg18_parentesco1_COUNT',
'agg18_parentesco12_COUNT',
'agg18_parentesco11_COUNT',
'agg18_parentesco10_COUNT',
'agg18_instlevel9_COUNT',
'agg18_parentesco6_COUNT',
'agg18_instlevel8_COUNT',
'agg18_instlevel7_COUNT',
'agg18_instlevel6_COUNT',
'agg18_estadocivil6_COUNT',
'agg18_parentesco8_COUNT',
'agg18_instlevel5_COUNT',
'agg18_instlevel4_COUNT',
'agg18_parentesco9_COUNT',
'agg18_instlevel3_COUNT',
'agg18_instlevel2_COUNT',
'dependency_yes',
'agg18_instlevel1_COUNT',
'agg18_parentesco3_COUNT',
'agg18_estadocivil7_COUNT',
'dependency_no'] + ['male', 'escolari', 'v18q', 'parentesco1'] + ['female', 'overcrowding', 'agg18_estadocivil2_COUNT'] + [
'hacdor',
'instlevel_LE',
'parentesco_LE',
'age',
'estadocivil_LE',
'dis',
'abastagua_LE',
'hhsize',
'v18q1',
'agg18_parentesco12_MEAN',
'rez_esc',
'refrig',
'agg18_parentesco11_MEAN',
'agg18_instlevel9_MEAN',
'agg18_estadocivil6_MEAN',
'agg18_instlevel6_MEAN',
'hogar_mayor',
'SQBage'] + [
'geo_pared_LE_0',
'geo_pared_LE_5',
'geo_manual_elec_LE_3',
'geo_energcocinar_LE_2',
'geo_hogar_mayor',
'geo_energcocinar_LE_0',
'geo_pared_LE_7',
'geo_pared_LE_6',
'agg18_parentesco10_MEAN',
'geo_energcocinar_LE_1',
'geo_pared_LE_3',
'geo_elimbasu_LE_4',
'geo_epared_LE_2',
'agg18_parentesco8_MEAN',
'geo_epared_LE_0',
'geo_etecho_LE_2',
'agg18_parentesco6_MEAN',
'agg18_parentesco4_MEAN',
'geo_manual_elec_LE_2'] + ['geo_sanitario_LE_3',
'geo_eviv_LE_0',
'agg18_parentesco7_MEAN',
'geo_manual_elec_LE_4',
'geo_pared_LE_2',
'geo_sanitario_LE_0',
'geo_pared_LE_4',
'geo_hogar_nin',
'geo_elimbasu_LE_2'] 
<drop_column>
coord_pickup = np.vstack(( train[['pickup_latitude', 'pickup_longitude']].values,
test[['pickup_latitude', 'pickup_longitude']].values))
coord_dropoff = np.vstack(( train[['dropoff_latitude', 'dropoff_longitude']].values,
test[['dropoff_latitude', 'dropoff_longitude']].values))
coords = np.hstack(( coord_pickup,coord_dropoff))
sample_ind = np.random.permutation(len(coords))
kmeans = MiniBatchKMeans(n_clusters=10, batch_size=10000 ).fit(coords[sample_ind])
 for df in(train,test):
df.loc[:, 'pickup_dropoff_loc'] = kmeans.predict(df[['pickup_latitude', 'pickup_longitude',
'dropoff_latitude','dropoff_longitude']] )
lgb_drop_cols = extra_drop_features + ["idhogar"]
try:
X_train.drop(lgb_drop_cols, axis=1, inplace=True)
 X_test.drop(lgb_drop_cols, axis=1, inplace=True)
 except:
print("Error dropping columns" )<compute_train_metric>
train = train.drop(['id', 'vendor_id', 'pickup_date','pickup_datetime'], axis = 1)
 train = train.drop(['dropoff_datetime','avg_speed_h','avg_speed_m', 'trip_duration'], axis = 1)
 Test_id = test['id']
test = test.drop(['id','vendor_id','pickup_date','pickup_datetime'], axis = 1 )
opt_parameters = {'colsample_bytree': 0.93, 'min_child_samples': 56, 'num_leaves': 19, 'subsample': 0.84}
def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) ,-1 ).argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', f1, True)
 fit_params={"early_stopping_rounds":300,
"eval_metric" : evaluate_macroF1_lgb,
"eval_set" : [(X_train,y_train),(X_test,y_test)],
'eval_names': ['train', 'valid'],
'verbose': False,
'categorical_feature': 'auto'}
def learning_rate_power_0997(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.995, current_iter)
 return max(lr, min_learning_rate)
 fit_params['verbose'] = 200
fit_params['callbacks'] = [lgb.reset_parameter(learning_rate=learning_rate_power_0997)]<train_model>
le=LabelEncoder()
train['store_and_fwd_flag'] = le.fit_transform(train['store_and_fwd_flag'])
 test['store_and_fwd_flag'] = le.transform(test['store_and_fwd_flag'] )
def _parallel_fit_estimator(estimator, X, y, sample_weight=None, **fit_params):

 if sample_weight is not None:
estimator.fit(X, y, sample_weight=sample_weight, **fit_params)
 else:
estimator.fit(X, y, **fit_params)
 return estimator
class VotingClassifierLGBM(VotingClassifier):

 def fit(self, X, y, sample_weight=None, **fit_params):
if isinstance(y, np.ndarray)and len(y.shape)> 1 and y.shape[1] > 1:
raise NotImplementedError('Multilabel and multi-output'
' classification is not supported.')
 if self.voting not in('soft', 'hard'):
raise ValueError("Voting must be 'soft' or 'hard'; got(voting=%r)"
% self.voting)
 if self.estimators is None or len(self.estimators)== 0:
raise AttributeError('Invalid `estimators` attribute, `estimators`'
' should be a list of(string, estimator)'
' tuples')
 if(self.weights is not None and
len(self.weights)!= len(self.estimators)) :
raise ValueError('Number of classifiers and weights must be equal'
'; got %d weights, %d estimators'
%(len(self.weights), len(self.estimators)))
 if sample_weight is not None:
for name, step in self.estimators:
if not has_fit_parameter(step, 'sample_weight'):
raise ValueError('Underlying estimator '%s' does not'
' support sample weights.' % name)
 names, clfs = zip(*self.estimators)
 self._validate_names(names)
 n_isnone = np.sum([clf is None for _, clf in self.estimators])
 if n_isnone == len(self.estimators):
raise ValueError('All estimators are None.At least one is '
'required to be a classifier!')
 self.le_ = LabelEncoder().fit(y)
 self.classes_ = self.le_.classes_
self.estimators_ = []
transformed_y = self.le_.transform(y)
 self.estimators_ = Parallel(n_jobs=self.n_jobs )(
 delayed(_parallel_fit_estimator )(clone(clf), X, transformed_y,
sample_weight=sample_weight, **fit_params)
 for clf in clfs if clf is not None)
 return self<train_model>
x = train.drop(['log_trip_duration'],1)
 y = train['log_trip_duration']
clfs = []
for i in range(10):
clf = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=314+i, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced')
 clf.set_params(**opt_parameters)
 clfs.append(( 'lgbm{}'.format(i), clf))
vc = VotingClassifierLGBM(clfs, voting='soft')
 del clfs
_ = vc.fit(X_train, y_train, **fit_params)
 clf_final = vc.estimators_[0]<compute_train_metric>
model_rnd_frst=RandomForestRegressor(n_estimators=10, criterion='mse', max_depth=None,
min_samples_split=2, min_samples_leaf=1,
min_weight_fraction_leaf=0.0, max_features='auto',
max_leaf_nodes=None, min_impurity_split=1e-07,
bootstrap=True, oob_score=False, n_jobs=-1,
random_state=None, verbose=1, warm_start=False)
 print(cross_val_score(model_rnd_frst,x,y))
global_score = f1_score(y_test, clf_final.predict(X_test), average='macro')
 vc.voting = 'soft'
global_score_soft = f1_score(y_test, vc.predict(X_test), average='macro')
 vc.voting = 'hard'
global_score_hard = f1_score(y_test, vc.predict(X_test), average='macro')
 print('Validation score of a single LGBM Classifier: {:.4f}'.format(global_score))
print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_score_soft))
print('Validation score of a VotingClassifier on 3 LGBMs with hard voting strategy: {:.4f}'.format(global_score_hard))<compute_test_metric>
Xtr, Xv, ytr, yv = train_test_split(x, y, test_size=0.2, random_state=1987)
 dtrain = xgb.DMatrix(Xtr, label=ytr)
 dvalid = xgb.DMatrix(Xv, label=yv)
 watchlist = [(dtrain, 'train'),(dvalid, 'valid')]
xgb_pars = {'min_child_weight': 50, 'eta': 0.3, 'colsample_bytree': 0.3, 'max_depth': 10,
'subsample': 0.8, 'lambda': 1., 'nthread': -1, 'booster' : 'gbtree', 'silent': 1,
'eval_metric': 'rmse', 'objective': 'reg:linear'}
model = xgb.train(xgb_pars, dtrain, 150, watchlist, early_stopping_rounds=100,
maximize=False, verbose_eval=10)
 print('Modeling RMSLE %.5f' % model.best_score )
print('Validation score of a single LGBM Classifier: {:.4f}'.format(global_score))
print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_score_soft))
print('Validation score of a VotingClassifier on 3 LGBMs with hard voting strategy: {:.4f}'.format(global_score_hard))<compute_test_metric>
ranked_features = feature_importance(clf_final, X_train )<create_dataframe>
dtest = xgb.DMatrix(test)
 pred = model.predict(dtest)
 pred = np.exp(pred)- 1
y_subm = pd.DataFrame()
y_subm['Id'] = test_ids<predict_on_test>