Kaggle competition: New York City Taxi Trip Duration
<SOS> metric: RMSLE Kaggle data source: new-york-city-taxi-trip-duration<sort_values>
%matplotlib inline
bool_features = [
col
for col in stacked_DF.select_dtypes(include=["number"] ).columns
if np.array_equal(
 np.sort(stacked_DF[col].unique() , axis=0), np.sort([0, 1], axis=0)
)
 ]
cat_features = [col for col in stacked_DF.select_dtypes(exclude=["number"] ).columns]
num_features = [
col
for col in stacked_DF.select_dtypes(include=["number"] ).columns
if col not in(bool_features)and col != "SalePrice"
]
bool_features.remove("Train")
 X_train_bool = stacked_DF.loc[stacked_DF["Train"] == 1, bool_features]
X_test_bool = stacked_DF.loc[stacked_DF["Train"] == 0, bool_features]
ohe_cols_a = []
ohe_cols_b = []
for col in cat_features:
if set(stacked_DF.loc[stacked_DF["Train"] == 1, col].unique())!= set(
 stacked_DF.loc[stacked_DF["Train"] == 0, col].unique() 
):
ohe_cols_b.append(col)
 ohe_cols_a = list(set(cat_features)- set(ohe_cols_b))<prepare_x_and_y>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
X_train = stacked_DF.loc[stacked_DF["Train"] == 1].drop(
 labels=["SalePrice", "Train"], axis=1 
)
 y_train = stacked_DF.loc[stacked_DF["Train"] == 1, "SalePrice"].apply(np.log)
 X_test = stacked_DF.loc[stacked_DF["Train"] == 0].drop(
 labels=["SalePrice", "Train"], axis=1 
)
 preprocessor = ColumnTransformer(
 transformers=[ 
("onehota", OneHotEncoder(sparse=False, drop="first"), ohe_cols_a), 
("onehotb", OneHotEncoder(sparse=False, handle_unknown="ignore"), ohe_cols_b), 
("scaler", StandardScaler() , num_features),
],
remainder="drop", 
)
 pipeline = Pipeline(
 [("Preprocessor", preprocessor),("VarThreshold", VarianceThreshold(0.01)) ] 
)
 X_train_preprocessed = pipeline.fit_transform(X_train)
 X_test_preprocessed = pipeline.transform(X_test)
 oh_encoded_a = list(
 preprocessor.named_transformers_.onehota.get_feature_names(ohe_cols_a)
)
 oh_encoded_b = list(
 preprocessor.named_transformers_.onehotb.get_feature_names(ohe_cols_b)
)
 oh_encoded_cols = oh_encoded_a + oh_encoded_b
feature_names = np.array(oh_encoded_cols + num_features, order="K")
 feature_names = feature_names[pipeline.named_steps.VarThreshold.get_support() ]
X_train_preprocessed = pd.DataFrame(data=X_train_preprocessed, columns=feature_names)
 X_test_preprocessed = pd.DataFrame(
 data=X_test_preprocessed, columns=feature_names, index=X_test_bool.index 
)
 X_train = pd.concat([X_train_bool, X_train_preprocessed], axis=1)
 X_test = pd.concat([X_test_bool, X_test_preprocessed], axis=1)
<train_model>
train.isna().sum()
model = Lasso(alpha=0.01)
 model.fit(X_train, y_train)
 feature_imp = permutation_importance(
 model, X_train, y_train, n_repeats=10, n_jobs=-1, random_state=random_state 
)
 perm_ft_imp_df = pd.DataFrame(
 data=feature_imp.importances_mean, columns=["FeatureImp"], index=X_train.columns
).sort_values(by="FeatureImp", ascending=False)
 model_ft_imp_df = pd.DataFrame(
 data=model.coef_, columns=["FeatureImp"], index=X_train.columns
).sort_values(by="FeatureImp", ascending=False)
 fig, ax = plt.subplots(2, 1, figsize=(12, 22))
perm_ft_imp_df_nonzero = perm_ft_imp_df[perm_ft_imp_df["FeatureImp"] != 0]
model_ft_imp_df_nonzero = model_ft_imp_df[model_ft_imp_df["FeatureImp"] != 0]
sns.barplot(
 x=perm_ft_imp_df_nonzero["FeatureImp"],
y=perm_ft_imp_df_nonzero.index,
ax=ax[0],
palette="vlag", 
)
 sns.barplot(
 x=model_ft_imp_df_nonzero["FeatureImp"],
y=model_ft_imp_df_nonzero.index,
ax=ax[1],
palette=sns.diverging_palette(10, 220, sep=2, n=80), 
)
 ax[0].set_title("Permutation Feature Importance")
 ax[1].set_title("Lasso Feature Importance")
 plt.show()<init_hyperparams>
Optimized parameters: {rf_params}")
<init_hyperparams>
train['trip_duration'] = np.log(train['trip_duration'].values)
Optimized parameters: {lgbm_params}")
<init_hyperparams>
rf_params = {"max_depth": 8, "max_features": 40, "n_estimators": 132}
svr_params = {
"kernel": "poly",
"C": 0.053677105521141605,
"epsilon": 0.03925943476562099,
"coef0": 0.9486751042886584,
}
ridge_params = {
"alpha": 0.9999189637151178,
"tol": 0.8668539399622242,
"solver": "cholesky",
}
lasso_params = {"alpha": 0.0004342843645993161, "selection": "random"}
lgbm_params = {
"num_leaves": 16,
"max_depth": 6,
"learning_rate": 0.16060612646519587,
"n_estimators": 64,
"min_child_weight": 0.4453842422224686,
}<compute_train_metric>
def haversine(lat1, lon1, lat2, lon2):
R = 6372800
phi1, phi2 = math.radians(lat1), math.radians(lat2)
 dphi = math.radians(lat2 - lat1)
 dlambda = math.radians(lon2 - lon1)
 a = math.sin(dphi/2)**2 + \
math.cos(phi1)*math.cos(phi2)*math.sin(dlambda/2)**2
return 2*R*math.atan2(math.sqrt(a), math.sqrt(1 - a))
cv = KFold(n_splits=4, random_state=random_state)
 svr = SVR(**svr_params)
 ridge = Ridge(**ridge_params, random_state=random_state)
 lasso = Lasso(**lasso_params, random_state=random_state)
 lgbm = LGBMRegressor(**lgbm_params, random_state=random_state)
 rf = RandomForestRegressor(**rf_params, random_state=random_state)
 stack = StackingCVRegressor(
 regressors=[svr, ridge, lasso, lgbm, rf],
meta_regressor=LinearRegression(n_jobs=-1),
random_state=random_state,
cv=cv,
n_jobs=-1, 
)
 svr_scores = cross_val_score(
 svr, X_train, y_train, cv=cv, n_jobs=-1, error_score="neg_root_mean_squared_error" 
)
 ridge_scores = cross_val_score(
 ridge, X_train, y_train, cv=cv, n_jobs=-1, error_score="neg_root_mean_squared_error" 
)
 lasso_scores = cross_val_score(
 lasso, X_train, y_train, cv=cv, n_jobs=-1, error_score="neg_root_mean_squared_error" 
)
 lgbm_scores = cross_val_score(
 lgbm, X_train, y_train, cv=cv, n_jobs=-1, error_score="neg_root_mean_squared_error" 
)
 rf_scores = cross_val_score(
 rf, X_train, y_train, cv=cv, n_jobs=-1, error_score="neg_root_mean_squared_error" 
)
 stack_scores = cross_val_score(
 stack, X_train, y_train, cv=cv, n_jobs=-1, error_score="neg_root_mean_squared_error" 
)
 scores = [svr_scores, ridge_scores, lasso_scores, lgbm_scores, rf_scores, stack_scores]
models = ["SVR", "RIDGE", "LASSO", "LGBM", "RF", "STACK"]
score_medians = [
round(np.median([mean for mean in modelscore]), 5)for modelscore in scores
] 
<save_to_csv>
train["distance"] = train.apply(lambda row: haversine(row["pickup_latitude"], row["pickup_longitude"], row["dropoff_latitude"], row["dropoff_longitude"]), axis=1)
 test["distance"] = test.apply(lambda row: haversine(row["pickup_latitude"], row["pickup_longitude"], row["dropoff_latitude"], row["dropoff_longitude"]), axis=1)
stack.fit(X_train.values, y_train.values)
 predictions = stack.predict(X_test.values)
 predictions = np.exp(predictions)
 submission = pd.DataFrame({"Id": submission_ID, "SalePrice": predictions})
 submission.to_csv("submission.csv", index=False )<categorify>
train['pickup_datetime'] = pd.to_datetime(train['pickup_datetime'])
 train['dropoff_datetime'] = pd.to_datetime(train['dropoff_datetime'])
 test['pickup_datetime'] = pd.to_datetime(test['pickup_datetime'])
 train['hour'] = train.pickup_datetime.dt.hour
train['day'] = train.pickup_datetime.dt.dayofweek
train['month'] = train.pickup_datetime.dt.month
test['hour'] = test.pickup_datetime.dt.hour
test['day'] = test.pickup_datetime.dt.dayofweek
test['month'] = test.pickup_datetime.dt.month
def label_encoding(train: pd.DataFrame, test: pd.DataFrame, col_definition: dict):

 n_train = len(train)
 train = pd.concat([train, test], sort=False ).reset_index(drop=True)
 for f in col_definition['encode_col']:
try:
lbl = preprocessing.LabelEncoder()
train[f] = lbl.fit_transform(list(train[f].values))
except:
print(f)
 test = train[n_train:].reset_index(drop=True)
 train = train[:n_train]
return train, test<load_from_csv>
y_train = train["trip_duration"]
X_train = train[["vendor_id","passenger_count","pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","month","hour","day","distance"]]
X_testdata = test[["vendor_id","passenger_count","pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","month","hour","day","distance"]]
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 sub = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv' )<count_unique_values>
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.linear_model import SGDRegressor
from sklearn.model_selection import ShuffleSplit
import xgboost as xgb
unique_cols = list(train.columns[train.nunique() == 1])
 print(unique_cols )<count_duplicates>
X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, random_state=42,test_size= 0.1)
duplicated_cols = list(train.columns[train.T.duplicated() ])
 print(duplicated_cols )<categorify>
rfr = RandomForestRegressor(n_estimators=100,min_samples_leaf=5, min_samples_split=50, max_depth=80,verbose=0,max_features="auto",n_jobs=-1)
 rfr.fit(X_train, y_train )
train, test = label_encoding(train, test, col_definition={'encode_col':categorical_cols} )<prepare_x_and_y>
train_pred = rfr.predict(X_testdata)
X_train = train.drop(['Id', 'SalePrice'], axis=1)
 y_train = np.log1p(train['SalePrice'])
 X_test = test.drop(['Id', 'SalePrice'], axis=1 )<prepare_x_and_y>
sample = pd.read_csv('.. /input/sample_submission.csv' )
y_preds_lgb = []
models_lgb = []
oof_train_lgb = np.zeros(( len(X_train),))
y_preds_xgb = []
models_xgb = []
oof_train_xgb = np.zeros(( len(X_train),))
y_preds = []
oof_train = np.zeros(( len(X_train),))
cv = KFold(n_splits=5, shuffle=True, random_state=0)
 params_lgb = {
'num_leaves': 29,
'max_depth': 6,
'objective': 'regression',
'metric': 'rmse',
'learning_rate': 0.05
}
params_xgb = {
'objective': 'reg:squarederror',
'eval_metric': 'rmse'
}
for fold_id,(train_index, valid_index)in enumerate(cv.split(X_train)) :
X_tr = X_train.loc[train_index, :]
X_val = X_train.loc[valid_index, :]
y_tr = y_train[train_index]
y_val = y_train[valid_index]
lgb_train = lgb.Dataset(X_tr,
y_tr,
categorical_feature=categorical_cols)
 lgb_eval = lgb.Dataset(X_val,
y_val,
reference=lgb_train,
categorical_feature=categorical_cols)
 model_lgb = lgb.train(params_lgb,
lgb_train,
valid_sets=[lgb_train, lgb_eval],
verbose_eval=10,
num_boost_round=1000,
early_stopping_rounds=10)
 oof_train_lgb[valid_index] = model_lgb.predict(X_val,
num_iteration=model_lgb.best_iteration)
 y_pred_lgb = model_lgb.predict(X_test,
num_iteration=model_lgb.best_iteration)
 y_preds_lgb.append(y_pred_lgb)
 models_lgb.append(model_lgb)
 dtrain = xgb.DMatrix(X_tr, label=y_tr)
 dvalid = xgb.DMatrix(X_val, label=y_val)
 dtest = xgb.DMatrix(X_test)
 evals = [(dtrain, 'train'),(dvalid, 'eval')]
model_xgb = xgb.train(params_xgb,
dtrain,
num_boost_round=1000,
early_stopping_rounds=10,
evals=evals 
)
 oof_train_xgb[valid_index] = model_xgb.predict(dvalid)
 y_pred_xgb = model_xgb.predict(dtest, ntree_limit=model_xgb.best_ntree_limit)
 y_preds_xgb.append(y_pred_xgb)
 models_xgb.append(model_xgb)
 y_preds_ensemble =(y_pred_lgb + y_pred_xgb)/ 2
y_preds.append(y_preds_ensemble)
 oof_train_ensemble =(oof_train_lgb[valid_index] + oof_train_xgb[valid_index])/ 2
oof_train[valid_index] = oof_train_ensemble<compute_test_metric>
my_submission = pd.DataFrame({'id': test.id, 'trip_duration': np.exp(train_pred)})
print(f'CV: {np.sqrt(mean_squared_error(y_train, oof_train)) }' )<prepare_output>
my_submission.to_csv('sub.csv', index=False )