Kaggle competition: New York City Taxi Fare Prediction
<SOS> metric: RMSE Kaggle data source: new-york-city-taxi-fare-prediction<categorify>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
encoder = DummiesEncoding(cat_features_ori=cat_features, dummy_na=True)
 X_train_cat_enc = encoder.fit_transform(X_train_cat)
 print(f'Shape before encoding: {X_train_cat.shape}')
 print(f'Shape after encoding: {X_train_cat_enc.shape}')
 X_train_cat_enc.head()<define_variables>
train_df = pd.read_csv('.. /input/train.csv', nrows = 5_000_000 )
TARGET = 'Survived'
INITIAL_FEATURES = ['Survived', 'Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked', 'name_title',
'ticket_class', 'cabin_class', 'name_length', 'age_cat', 'fare_cat', 'family_size']
INITIAL_PRED_FEATURES = [col for col in INITIAL_FEATURES if col not in TARGET]
DTYPE_MODIFICATION_DICT = {'Pclass': str}
NUM_FEATURES = ['Age', 'SibSp', 'Parch', 'Fare', 'name_length', 'family_size']
CAT_FEATURES = ['Pclass', 'Sex', 'Embarked', 'name_title', 'ticket_class', 'cabin_class', 'age_cat', 'fare_cat']
CAT_FEATURES_FINAL = ['Pclass_1', 'Pclass_2', 'Pclass_3', 'Pclass_nan', 'Sex_female', 'Sex_male', 'Sex_nan',
'Embarked_C', 'Embarked_Q', 'Embarked_S', 'Embarked_nan', 'name_title_Master.',
'name_title_Miss.', 'name_title_Mr.', 'name_title_Mrs.', 'name_title_OTHER',
'name_title_nan', 'ticket_class_A', 'ticket_class_OTHER', 'ticket_class_PC',
'ticket_class_SC', 'ticket_class_STON_SOTON', 'ticket_class_nan', 'cabin_class_A',
'cabin_class_B', 'cabin_class_C', 'cabin_class_D', 'cabin_class_E', 'cabin_class_FGT',
'cabin_class_nan', 'age_cat_0_10', 'age_cat_10_20', 'age_cat_20_40', 'age_cat_40_60',
'age_cat_greater_60', 'age_cat_nan', 'fare_cat_0_8', 'fare_cat_15_25',
'fare_cat_25_50', 'fare_cat_8_15', 'fare_cat_greater_50', 'fare_cat_nan']
MODEL_FEATURES = NUM_FEATURES + CAT_FEATURES_FINAL
NUM_STRATEGY_IMPUTER = 'median'
SCALER_TYPE = None
LOG_APPLICATION = False
COLS_TO_LOG = ['Fare', 'Age']
ENCODER_DUMMY_NA = True
NAME_TITLE = True
CABIN_CLASS = True
TICKET_CLASS = True
NAME_LENGTH = True
AGE_CAT = True
FARE_CAT = True
FAMILY_SIZE = True
initial_train_pipeline = Pipeline([ 
('feature_adder', CustomFeaturesTitanic(name_title=NAME_TITLE, cabin_class=CABIN_CLASS,
ticket_class=TICKET_CLASS, name_length=NAME_LENGTH,
age_cat=AGE_CAT, fare_cat=FARE_CAT, family_size=FAMILY_SIZE)) , 
('col_filter', FiltraColunas(features=INITIAL_FEATURES)) , 
('dtype_modifier', ModificaTipoPrimitivo(mod_dict=DTYPE_MODIFICATION_DICT)) , 
('dup_dropper', EliminaDuplicatas())
 ])
 initial_pred_pipeline = Pipeline([ 
('feature_adder', CustomFeaturesTitanic(name_title=NAME_TITLE, cabin_class=CABIN_CLASS,
ticket_class=TICKET_CLASS, name_length=NAME_LENGTH,
age_cat=AGE_CAT, fare_cat=FARE_CAT, family_size=FAMILY_SIZE)) , 
('col_filter', FiltraColunas(features=INITIAL_PRED_FEATURES)) , 
('dtype_modifier', ModificaTipoPrimitivo(mod_dict=DTYPE_MODIFICATION_DICT))
])
 num_pipeline = Pipeline([ 
('imputer', SimpleImputer(strategy=NUM_STRATEGY_IMPUTER)) , 
('log_transformer', DynamicLogTransformation(application=LOG_APPLICATION, num_features=NUM_FEATURES,
cols_to_log=COLS_TO_LOG)) , 
('scaler', DynamicScaler(scaler_type=SCALER_TYPE))
])
 cat_pipeline = Pipeline([ 
('encoder', DummiesEncoding(dummy_na=ENCODER_DUMMY_NA, cat_features_final=CAT_FEATURES_FINAL))
])
 prep_pipeline = ColumnTransformer([ 
('num', num_pipeline, NUM_FEATURES), 
('cat', cat_pipeline, CAT_FEATURES)
 ] )<split>
def haversine_np(lon1, lat1, lon2, lat2):

 lon1, lat1, lon2, lat2 = map(np.radians, [lon1, lat1, lon2, lat2])
 dlon = lon2 - lon1
dlat = lat2 - lat1
a = np.sin(dlat/2.0)**2 + np.cos(lat1)* np.cos(lat2)* np.sin(dlon/2.0)**2
c = 2 * np.arcsin(np.sqrt(a))
km = 6367 * c
return km
df = pd.read_csv(os.path.join(DATA_PATH, TRAIN_FILENAME))
df_prep = initial_train_pipeline.fit_transform(df)
 X_train, X_val, y_train, y_val = train_test_split(df_prep.drop(TARGET, axis=1), df_prep[TARGET].values,
test_size=.20, random_state=42)
 X_train_prep = prep_pipeline.fit_transform(X_train)
 X_val_prep = prep_pipeline.fit_transform(X_val)
 print(f'Shape of X_train_prep: {X_train_prep.shape}')
 print(f'Shape of X_val_prep: {X_val_prep.shape}')
 print(f'
Total features considered: {len(MODEL_FEATURES)}' )<create_dataframe>
train_df['distance'] = haversine_np(train_df['pickup_longitude'], train_df['pickup_latitude'],
train_df['dropoff_longitude'], train_df['dropoff_latitude'] )
df_prep = pd.DataFrame(X_train_prep, columns=MODEL_FEATURES)
 df_prep['Survived'] = y_train
plot_corr_matrix(df=df_prep, corr_col='Survived', figsize=(12, 12), cbar=False, n_vars=15 )<choose_model_class>
train_df['pickup_datetime'] = pd.to_datetime(train_df['pickup_datetime'] )
dtree = DecisionTreeClassifier()
forest = RandomForestClassifier()
lgbm = LGBMClassifier()
xgb = XGBClassifier()
adaboost = AdaBoostClassifier()
gradboost = GradientBoostingClassifier()
model_obj = [dtree, forest, lgbm, xgb, adaboost, gradboost]
model_names = [type(model ).__name__ for model in model_obj]
set_classifiers = {name: {'model': obj, 'params': {}} for(name, obj)in zip(model_names, model_obj)}
print(f'Classifiers that will be trained on next steps:
{model_names}' )<train_model>
train_df['year'] = train_df['pickup_datetime'].dt.year
train_df['month'] = train_df['pickup_datetime'].dt.month
train_df['day'] = train_df['pickup_datetime'].dt.day
train_df['hour'] = train_df['pickup_datetime'].dt.hour
train_df['minute'] = train_df['pickup_datetime'].dt.minute
trainer = ClassificadorBinario()
trainer.fit(set_classifiers, X_train_prep, y_train )<compute_test_metric>
print('Old size: %d' % len(train_df))
train_df = train_df.dropna(how = 'any', axis = 'rows')
 print('New size: %d' % len(train_df))
metrics = trainer.evaluate_performance(X_train_prep, y_train, X_val_prep, y_val)
 metrics<train_model>
cond = True
for col in {'pickup_latitude', 'pickup_longitude', 'dropoff_latitude', 'dropoff_longitude'}:
cond &= abs(train_df[col] - train_df[col].mean())< 5
full_trainer = ClassificadorBinario()
full_trainer.training_flow(set_classifiers, X_train_prep, y_train, X_val_prep, y_val,
features=MODEL_FEATURES, random_search=True)
 full_trainer.visual_analysis(features=MODEL_FEATURES, model_shap='LGBMClassifier' )<load_from_csv>
for col in {'pickup_latitude', 'pickup_longitude', 'dropoff_latitude', 'dropoff_longitude'}:
train_df['rough' + col] = train_df[col].round(2 )
metrics = pd.read_csv('output/metrics/metrics.csv')
 metrics<define_search_space>
a=train_df.groupby(['roughpickup_latitude','roughpickup_longitude'])[['pickup_latitude', 'pickup_longitude']].agg(['mean','count'] )
forest_tunning_grid = {
'bootstrap': [True, False],
'class_weight': [None, 'balanced'],
'criterion': ['gini', 'entropy'],
'max_depth': [5, 6, 7, 9, 10],
'n_estimators': np.arange(300, 600, 50),
'random_state': [42]
}
lgbm_tunning_grid = {
'boosting_type': ['gbdt'],
'class_weight': [None, 'balanced'],
'learning_rate': [0.003, 0.01, 0.03, 0.1, 0.3, 1, 3],
'max_depth': np.arange(-1, 100, 2),
'n_estimators': np.arange(300, 700, 50),
'num_leaves': [5, 10, 15, 20],
'objective': ['binary'],
'random_state': [42],
'reg_alpha': np.linspace (.0, 25, 15)
 }
xgboost_tunning_grid = {
'booster': ['gbtree'],
'max_depth': [3, 4, 6, 7],
'learning_rate': [0.003, 0.01, 0.03, 0.1, 0.3, 1, 3],
'n_estimators': np.arange(300, 700, 50),
'objective': ['binary:logistic'],
'seed': [42],
'reg_alpha': np.linspace (.0, 25, 15),
'reg_lambda': np.linspace (.0, 25, 15),
'colsample_bylevel': [0.5, 0.7, 0.9]
}
adaboost_tunning_grid = {
'base_estimator': [DecisionTreeClassifier(max_depth=7)],
'n_estimators': np.arange(50, 700, 50),
'learning_rate': [0.003, 0.01, 0.03, 0.1, 0.3, 1, 3],
'algorithm': ['SAMME', 'SAMME.R'],
'random_state': [42]
}
gradboost_tunning_grid = {
'n_estimators': np.arange(50, 700, 50),
'max_depth': [3, 6, 7, 8, 9, 10, 15],
'max_features': [None, 'auto', 'sqrt', 'log2'],
'max_leaf_nodes': np.arange(3, 50, 2),
'random_state': [42]
}<define_variables>
a.columns = ['mean_pickup_latitude','c1','mean_pickup_longitude','c2']
a.head()
a.sort_values(['c1','c2'],ascending=False ).head()
tunning_models_keys = ['RandomForestClassifier', 'LGBMClassifier', 'XGBClassifier', 'AdaBoostClassifier',
'GradientBoostingClassifier']
tunning_param_grids = [forest_tunning_grid, lgbm_tunning_grid, xgboost_tunning_grid, adaboost_tunning_grid,
gradboost_tunning_grid]
tunned_pipelines = {}
general_metrics = pd.DataFrame({})
 pipe_model_key = 'model'
X = X_train.append(X_val)
 y = np.concatenate(( y_train, y_val))
for model_name, param_grid in zip(tunning_models_keys, tunning_param_grids):
baseline_model = trainer._get_estimator(model_name)
 feature_importance = baseline_model.feature_importances_
general_metrics = general_metrics.append(trainer._get_metrics(model_name))
tunning_pipeline = Pipeline([ 
('prep', prep_pipeline), 
('selector', SeletorTopFeatures(feature_importance, k=len(MODEL_FEATURES))), 
(pipe_model_key, baseline_model)
 ])
 model_param_grid = {pipe_model_key + '__' + k: v for k, v in param_grid.items() }
tunning_param_grid = {
'prep__num__imputer__strategy': ['mean', 'median', 'most_frequent'],
'prep__num__log_transformer__application': [True, False],
'prep__num__scaler__scaler_type': [None, 'Standard', 'MinMax'],
'selector__k': np.arange(5, len(MODEL_FEATURES)+ 1, 2)
 }
tunning_param_grid.update(model_param_grid)
 tunning_search = RandomizedSearchCV(tunning_pipeline, tunning_param_grid, scoring='accuracy', cv=5,
n_jobs=-1, verbose=-1, random_state=42)
 tunning_search.fit(X, y)
 print(f'
Best hyperparameters for {model_name} found by RandomizedSearch:
')
 for k, v in tunning_search.best_params_.items() :
print(f'{k}: {v}')
 final_pipeline = tunning_search.best_estimator_
tunned_pipelines[model_name] = final_pipeline
final_metrics = clf_cv_performance(final_pipeline, X, y, model_name=model_name)
 model_metrics = trainer._get_metrics(model_name=model_name)
 metrics_cols = general_metrics.columns
final_metrics = final_metrics.loc[:, metrics_cols]
general_metrics = general_metrics.append(final_metrics)
 general_metrics<define_variables>
b=train_df.groupby(['roughdropoff_latitude','roughdropoff_longitude'])[['dropoff_latitude', 'dropoff_longitude']].agg(['mean','count'] )
FINAL_MODEL = 'RandomForestClassifier'
final_pipeline = tunned_pipelines[FINAL_MODEL]<load_from_csv>
b.columns = ['mean_dropoff_latitude','c1','mean_dropoff_longitude','c2']
b.head()
b.sort_values(['c1','c2'],ascending=False ).head(n=20)
df_test = pd.read_csv(os.path.join(DATA_PATH, TEST_FILENAME))
print(f'Shape of test dataset: {df_test.shape}')
 df_test.head()<categorify>
b[ b['mean_dropoff_latitude']<40.7].sort_values(['c1','c2'],ascending=False ).head()
model_consumer = ConsumoModelo(model=final_pipeline, features=INITIAL_PRED_FEATURES)
 prediction_pipeline = Pipeline([ 
('initial', initial_pred_pipeline), 
('prediction', model_consumer)
 ])
 df_pred = prediction_pipeline.fit_transform(df_test)
 df_pred.head()<save_to_csv>
a = a[['c1']].reset_index().rename(columns={'c1':'pickup_busyness'})
 b = b[['c1']].reset_index().rename(columns={'c1':'dropoff_busyness'} )
df_sub = df_test.merge(df_pred, how='left', left_index=True, right_index=True)
 df_sub = df_sub.loc[:, ['PassengerId', 'y_pred']]
df_sub.columns = ['PassengerId', 'Survived']
df_sub.to_csv('output/submission.csv', index=False)
 df_sub.head()<import_modules>
train_df = pd.merge(train_df,a, how='left')
 train_df = pd.merge(train_df,b, how='left' )
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns<load_from_csv>
X = train_df[['distance','year','month','day','hour','pickup_busyness','dropoff_busyness']].values
Y = train_df['fare_amount'].values
train_data = pd.read_csv('.. /input/titanic/train.csv')
 test_data = pd.read_csv('.. /input/titanic/test.csv')
 train = train_data.copy()
test = test_data.copy()<drop_column>
from sklearn.ensemble import RandomForestRegressor
train.drop(['PassengerId'], axis=1, inplace=True)
 test.drop(['PassengerId'], axis=1, inplace=True)
 pred = train['Survived']<count_missing_values>
from sklearn.ensemble import RandomForestRegressor
train.isnull().sum()<count_missing_values>
kwargs = {'bootstrap': True,
'max_depth': None,
'max_features': 3,
'min_samples_leaf': 9,
'min_samples_split': 2}
rand_regr = RandomForestRegressor(n_estimators=20, **kwargs )
train.isnull().sum()<count_missing_values>
rand_regr.fit(X, Y )
test.isnull().sum()<count_missing_values>
y_pred = rand_regr.predict(X)
 print('chi squared rand forest with date %s' %(np.sum(( Y-y_pred)**2.) /len(Y)) **0.5 )
test.isnull().sum()<data_type_conversions>
rand_regr.score(X,Y )
train['Age'].fillna(train['Age'].quantile(0.5), inplace=True)
 test['Age'].fillna(test['Age'].quantile(0.5), inplace=True )<data_type_conversions>
test_df = pd.read_csv('.. /input/test.csv' )
train['Embarked'].fillna('S', inplace=True)
 test['Embarked'].fillna('S', inplace=True )<data_type_conversions>
test_df['distance'] = haversine_np(test_df['pickup_longitude'], test_df['pickup_latitude'],
test_df['dropoff_longitude'], test_df['dropoff_latitude'] )
test['Fare'].fillna(test['Fare'].quantile(0.5), inplace=True )<categorify>
test_df['pickup_datetime'] = pd.to_datetime(test_df['pickup_datetime'] )
sex1 = pd.get_dummies(train['Sex'])
 sex2 = pd.get_dummies(test['Sex'] )<concatenate>
test_df['year'] = test_df['pickup_datetime'].dt.year
test_df['month'] = test_df['pickup_datetime'].dt.month
test_df['day'] = test_df['pickup_datetime'].dt.day
test_df['hour'] = test_df['pickup_datetime'].dt.hour
test_df['minute'] = test_df['pickup_datetime'].dt.minute
train.drop(['Sex'], axis=1, inplace=True)
 test.drop(['Sex'], axis=1, inplace=True)
 train = pd.concat([train, sex1], axis=1)
 test = pd.concat([test, sex2], axis=1 )<drop_column>
for col in {'pickup_latitude', 'pickup_longitude', 'dropoff_latitude', 'dropoff_longitude'}:
test_df['rough' + col] = test_df[col].round(2 )
train.drop(['female'], axis=1, inplace=True)
 test.drop(['female'], axis=1, inplace=True )<categorify>
test_df = pd.merge(test_df,a, how='left')
 test_df = pd.merge(test_df,b, how='left' )
embark1 = pd.get_dummies(train['Embarked'])
 embark2 = pd.get_dummies(test['Embarked'])
 train.drop(['Embarked'], axis=1, inplace=True)
 test.drop(['Embarked'], axis=1, inplace=True)
 train = pd.concat([train, embark1], axis=1)
 test = pd.concat([test, embark2], axis=1 )<feature_engineering>
test_df['pickup_busyness'] = test_df['pickup_busyness'].fillna(1)
 test_df['dropoff_busyness'] = test_df['dropoff_busyness'].fillna(1 )
def family(x):
if x['SibSp'] + x['Parch'] > 1:
return 1
else:
return 0
train['Family'] = train.apply(family, axis=1)
 test['Family'] = test.apply(family, axis=1 )<drop_column>
X_to_pred = test_df[['distance','year','month','day','hour','pickup_busyness','dropoff_busyness']].values
y_pred = rand_regr.predict(X_to_pred )
train.drop(['SibSp','Parch'], axis=1, inplace=True)
 test.drop(['SibSp','Parch'], axis=1, inplace=True )<feature_engineering>
submission = pd.DataFrame(
 {'key': test_df.key, 'fare_amount': y_pred},
columns = ['key', 'fare_amount'])
 submission.to_csv('submission.csv', index = False )
<categorify><EOS>