Kaggle competition: Titanic - Machine Learning from Disaster
oof_pred_avg = oof_pred_lgb*0.6 + oof_pred_xgb*0.4
oof_score_avg = mean_squared_error(y, oof_pred_avg, squared=False)
 print(f"oof score avg: {oof_score_avg:.5f}" )<define_variables>
train = pd.read_csv(".. /input/train.csv")
 train.head()
test_pred_avg = test_pred_lgb*0.6 + test_pred_xgb*0.4<save_to_csv>
train = train.drop(columns = ['Cabin','Name','Ticket','PassengerId'] )
sub = smpl_sub.copy()
sub[TGT_COL] = test_pred_avg
sub.to_csv("submission.csv", index=False)
 sub.head()<install_modules>
train['Age'].fillna(( train['Age'].mean()), inplace=True )
!pip install python-vivid<import_modules>
train[["Parch", "Survived"]].groupby(['Parch'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
import seaborn as sns
import matplotlib.pyplot as plt
from vivid.estimators.base import MetaBlock
from vivid.estimators.boosting.mixins import BoostingEarlyStoppingMixin
from catboost import CatBoostRegressor<import_modules>
train[["SibSp", "Survived"]].groupby(['SibSp'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
from vivid.features.base import BinningCountBlock
from vivid.features.base import CountEncodingBlock
from vivid.features.base import FilterBlock
from vivid.estimators.boosting import XGBRegressorBlock
from vivid.estimators.boosting import LGBMRegressorBlock
from vivid.estimators.boosting.block import create_boosting_seed_blocks
from vivid.estimators.linear import TunedRidgeBlock
from vivid.estimators.svm import SVRBlock
from vivid.estimators.ensumble import RFRegressorBlock
from vivid.estimators.base import EnsembleBlock, BaseBlock<load_from_csv>
yf = train.Survived
base_features = ['Parch',
'SibSp','Age', 'Fare','Pclass']
Xf = train[base_features]
train_X, val_X, train_y, val_y = train_test_split(Xf, yf, random_state=1)
 first_model = RandomForestRegressor(n_estimators=21, random_state=1 ).fit(train_X, train_y )
train_df = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/train.csv')
 test_df = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/test.csv')
 sample_submission_df = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/sample_submission.csv')
 feature_columns = [
'cont1',
'cont2',
'cont3',
'cont4',
'cont5',
'cont6',
'cont7',
'cont8',
'cont9',
'cont10',
'cont11',
'cont12',
'cont13',
'cont14',
]
y = train_df['target'].values<train_model>
train['FamilySize'] = train['SibSp'] + train['Parch']
train[['FamilySize', 'Survived']].groupby(['FamilySize'], as_index=False ).agg('mean' )
class PCABlock(BaseBlock):
def __init__(self, n_components=3, columns=None, *args, **kwrgs):
self.n_components = n_components
if columns is None: columns = feature_columns
self.columns = columns
super().__init__(name='pca_n={}'.format(n_components), *args, **kwrgs)
 def fit(self, source_df, y, experiment=None)-> pd.DataFrame:
clf = PCA(n_components=self.n_components)
 clf.fit(source_df[self.columns].values)
 self.clf_ = clf
return self.transform(source_df)
 def transform(self, source_df):
z = self.clf_.transform(source_df[self.columns])
 out_df = pd.DataFrame(z)
 return out_df.add_prefix('PCA_')
 class GaussianMixtureBlock(BaseBlock):
def __init__(self, n_components=3, columns=None, *args, **kwrgs):
self.n_components = n_components
if columns is None: columns = feature_columns
self.columns = columns
super().__init__(name='GMM_n={}'.format(n_components), *args, **kwrgs)
 def fit(self, source_df, y, experiment=None)-> pd.DataFrame:
clf = GaussianMixture(n_components=self.n_components)
 clf.fit(source_df[self.columns].values)
 self.clf_ = clf
return self.transform(source_df)
 def transform(self, source_df):
z = self.clf_.predict_proba(source_df[self.columns])
 z = np.clip(z, 1e-6,1 - 1e-6)
 out_df = pd.DataFrame(z)
 return out_df.add_prefix('GMM_' )<import_modules>
train['IsAlone'] = 0
train.loc[train['FamilySize'] == 0, 'IsAlone'] = 1
train[['IsAlone', 'Survived']].groupby(['IsAlone'], as_index=False ).mean()
from vivid.estimators.boosting.mixins import TunedBoostingBlock
from vivid.estimators.boosting.helpers import get_boosting_parameter_suggestions
from vivid.estimators.boosting.lgbm import LGBMRegressorBlock
import lightgbm as lgbm
from vivid.runner import create_runner<choose_model_class>
train[["Fare", "Survived"]].groupby(['Survived'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
class TunedLightGBMRegressorBlock(TunedBoostingBlock):
model_class = lgbm.LGBMRegressor
default_eval_metric = 'rmse'
initial_params = LGBMRegressorBlock.initial_params
def generate_model_class_try_params(self, trial):
param = get_boosting_parameter_suggestions(trial)
 param['n_jobs'] = -1
return param<choose_model_class>
train.groupby(['Sex','Survived'])[['Fare']].agg(['min','mean','max'] )
feature_blocks = [
BinningCountBlock(name='BINS', column=feature_columns),
CountEncodingBlock(name='CE', column=feature_columns),
FilterBlock(name='F', column=feature_columns),
PCABlock(n_components=3),
GaussianMixtureBlock(n_components=3)
 ]
runner = create_runner(blocks=[
LGBMRegressorBlock(name='normal_lgbm', parent=feature_blocks),
TunedLightGBMRegressorBlock(name='tuned_lgbm', parent=feature_blocks, n_trials=50)
 ] )<train_model>
train.loc[ train['Fare'] <= 7.22, 'Fare'] = 0
train.loc[(train['Fare'] > 7.22)&(train['Fare'] <= 21.96), 'Fare'] = 1
train.loc[(train['Fare'] > 21.96)&(train['Fare'] <= 40.82), 'Fare'] = 2
train.loc[ train['Fare'] > 40.82, 'Fare'] = 3
train['Fare'] = train['Fare'].astype(int )
oof_results = runner.fit(train_df[feature_columns], y=y )<predict_on_test>
train.loc[ train['Age'] <= 16, 'Age'] = 1
train.loc[(train['Age'] > 16)&(train['Age'] <= 32), 'Age'] = 2
train.loc[(train['Age'] > 32)&(train['Age'] <= 64), 'Age'] = 3
train.loc[ train['Age'] > 64, 'Age'] = 4
train['Age'] = train['Age'].astype(int )
test_results = runner.predict(test_df )<create_dataframe>
train['Age*Class'] = train.Age * train.Pclass
oof_df = pd.DataFrame()
for result in oof_results:
oof_df[result.block.name] = result.out_df.values[:, 0]<create_dataframe>
train[["Age*Class", "Survived"]].groupby(['Age*Class'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
score_df = pd.DataFrame()
for name, pred in oof_df.T.iterrows() :
score_i = regression_metrics(y, pred)
 score_df = score_df.append(pd.Series(score_i, name=name))<sort_values>
y2 = train.Survived
base_features2 = ['Parch','SibSp','Age', 'Fare','Pclass','Age*Class','FamilySize','IsAlone']
X2 = train[base_features2]
train_X2, val_X2, train_y2, val_y2 = train_test_split(X2, y2, random_state=1)
 second_model = RandomForestRegressor(n_estimators=21, random_state=1 ).fit(train_X2, train_y2)
 inter2 = pdp.pdp_interact(model=second_model, dataset=val_X2, model_features=base_features2, features=['Age', 'Pclass'])
 pdp.pdp_interact_plot(pdp_interact_out=inter2, feature_names=['Age', 'Pclass'], plot_type='contour')
 plt.show()
score_df.sort_values('rmse' )<save_to_csv>
dummies_Sex = pd.get_dummies(train.Sex)
 dummies_Embarked = pd.get_dummies(train.Embarked)
 train_ready = pd.concat([train, dummies_Sex,dummies_Embarked], axis=1)
 train_ready.head()
for result in test_results:
out_df = result.out_df
sub_df = sample_submission_df.copy()
sub_df['target'] = result.out_df.values[:, 0]
to = f'/kaggle/working/{str(result.block.name)}_submission.csv'
print('save to {}'.format(to))
sub_df.to_csv(to, index=False )<load_from_csv>
train_ready = train_ready.drop(columns = ['Sex','Embarked'] )
train = pd.read_csv(input_path / 'train.csv', index_col='id')
 display(train.head() )<compute_test_metric>
for name in train_ready:
print(name, "column entropy :", round(stats.entropy(train_ready[name].value_counts(normalize=True), base=2),2))
z = np.abs(stats.zscore(train))
print(z )<filter>
test = pd.read_csv(".. /input/test.csv")
 test = test.drop(columns = ['Cabin','Name','Ticket','PassengerId'])
 test.head()
print(len(train))
train = train[(z < 3 ).all(axis=1)]
print(len(train))<load_from_csv>
test['Age'].fillna(( test['Age'].mean()), inplace=True)
 test['Fare'].fillna(( test['Fare'].mean()), inplace=True )
test = pd.read_csv(input_path / 'test.csv', index_col='id' )<feature_engineering>
test.loc[ test['Fare'] <= 7.22, 'Fare'] = 0
test.loc[(test['Fare'] > 7.22)&(test['Fare'] <= 21.96), 'Fare'] = 1
test.loc[(test['Fare'] > 21.96)&(test['Fare'] <= 40.82), 'Fare'] = 2
test.loc[ test['Fare'] > 40.82, 'Fare'] = 3
for df in [train, test]:
df['sum']=train.sum(axis=0)
 df['min'] = train[features].min(axis=0)
 df['max'] = train[features].max(axis=0)
 df['mean'] = train[features].mean(axis=0)
 df['median'] = train[features].median(axis=0)
 df['skew'] = train[features].skew(axis=0)
 df['kurtosis'] = train[features].kurtosis(axis=0)
 display(test.head() )<load_from_csv>
test['FamilySize'] = test['SibSp'] + test['Parch'] + 1
test['IsAlone'] = 0
test.loc[test['FamilySize'] == 1, 'IsAlone'] = 1
submission = pd.read_csv(input_path / 'sample_submission.csv', index_col='id')
 display(submission.head() )<split>
test.loc[ test['Age'] <= 16, 'Age'] = 1
test.loc[(test['Age'] > 16)&(test['Age'] <= 32), 'Age'] = 2
test.loc[(test['Age'] > 32)&(test['Age'] <= 64), 'Age'] = 3
test.loc[ test['Age'] > 64, 'Age'] = 4
target = train.pop('target')
 X_train, X_test, y_train, y_test = train_test_split(train, target, train_size=0.90, random_state=42 )<train_model>
test['Age*Class'] = test.Age * test.Pclass
params={
'n_estimators': 300,
'subsample': 0.7,
'colsample_bytree': 0.7,
'eta': 0.035282124803629786,
'max_depth': 11,
'min_child_weight': 10,
'tree_method':'gpu_hist',
'random_state':42
}
model = xgb.XGBRegressor(**params)
 model.fit(X_train, y_train)
 preds = model.predict(X_test)
 print(mean_squared_error(preds, y_test)**0.5 )<save_to_csv>
test_dummies_Sex = pd.get_dummies(test.Sex)
 test_dummies_Embarked = pd.get_dummies(test.Embarked)
 test_ready = pd.concat([test, test_dummies_Sex,test_dummies_Embarked], axis=1)
 test_ready.head()
model.fit(train, target)
 submission['target'] = model.predict(test)
 submission.to_csv('xgb_regression.csv' )<import_modules>
test_ready = test_ready.drop(columns = ['Sex','Embarked'] )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import gc, math
from pandas.tseries.holiday import USFederalHolidayCalendar as calendar
from sklearn.metrics import mean_squared_error
import lightgbm as lgb
from sklearn.model_selection import KFold, StratifiedKFold, GroupKFold
from tqdm import tqdm
from sklearn.preprocessing import LabelEncoder<set_options>
for name in test_ready:
print(name, "column entropy :",round(stats.entropy(test_ready[name].value_counts(normalize=True), base=2),2))
sns.set(rc={'figure.figsize':(11,8)})
 sns.set(style="whitegrid" )<load_from_csv>
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
%%time
metadata_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/building_metadata.csv')
 train_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/train.csv', parse_dates=['timestamp'])
 test_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/test.csv', parse_dates=['timestamp'])
 weather_train_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/weather_train.csv', parse_dates=['timestamp'])
 weather_test_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/weather_test.csv', parse_dates=['timestamp'] )<feature_engineering>
y = train_ready['Survived'].values
X = train_ready.drop('Survived',axis=1 ).values
weather = pd.concat([weather_train_df,weather_test_df],ignore_index=True)
 weather_key = ['site_id', 'timestamp']
temp_skeleton = weather[weather_key + ['air_temperature']].drop_duplicates(subset=weather_key ).sort_values(by=weather_key ).copy()
temp_skeleton['temp_rank'] = temp_skeleton.groupby(['site_id', temp_skeleton.timestamp.dt.date])['air_temperature'].rank('average')
 df_2d = temp_skeleton.groupby(['site_id', temp_skeleton.timestamp.dt.hour])['temp_rank'].mean().unstack(level=1)
 site_ids_offsets = pd.Series(df_2d.values.argmax(axis=1)- 14)
 site_ids_offsets.index.name = 'site_id'
def timestamp_align(df):
df['offset'] = df.site_id.map(site_ids_offsets)
 df['timestamp_aligned'] =(df.timestamp - pd.to_timedelta(df.offset, unit='H'))
df['timestamp'] = df['timestamp_aligned']
del df['timestamp_aligned']
return df<concatenate>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state=21, stratify=y )
weather_train_df = timestamp_align(weather_train_df)
 weather_test_df = timestamp_align(weather_test_df )<drop_column>
warnings.filterwarnings("ignore")
del weather
del df_2d
del temp_skeleton
del site_ids_offsets<count_missing_values>
clfs = []
seed = 3
clfs.append(( "LogReg",
Pipeline([("Scaler", StandardScaler()), 
("LogReg", LogisticRegression())])))
 clfs.append(( "XGBClassifier",
Pipeline([("Scaler", StandardScaler()), 
("XGB", XGBClassifier())])))
 clfs.append(( "KNN",
Pipeline([("Scaler", StandardScaler()), 
("KNN", KNeighborsClassifier(n_neighbors=8)) ])))
 clfs.append(( "DecisionTreeClassifier",
Pipeline([("Scaler", StandardScaler()), 
("DecisionTrees", DecisionTreeClassifier())])))
 clfs.append(( "RandomForestClassifier",
Pipeline([("Scaler", StandardScaler()), 
("RandomForest", RandomForestClassifier())])))
 clfs.append(( "GradientBoostingClassifier",
Pipeline([("Scaler", StandardScaler()), 
("GradientBoosting", GradientBoostingClassifier(n_estimators=100)) ])))
 clfs.append(( "RidgeClassifier",
Pipeline([("Scaler", StandardScaler()), 
("RidgeClassifier", RidgeClassifier())])))
 clfs.append(( "BaggingRidgeClassifier",
Pipeline([("Scaler", StandardScaler()), 
("BaggingClassifier", BaggingClassifier())])))
 clfs.append(( "ExtraTreesClassifier",
Pipeline([("Scaler", StandardScaler()), 
("ExtraTrees", ExtraTreesClassifier())])))
 scoring = 'accuracy'
n_folds = 7
results, names = [], []
for name, model in clfs:
kfold = KFold(n_splits=n_folds, random_state=seed)
 cv_results = cross_val_score(model, X_train, y_train,
cv= 5, scoring=scoring,
n_jobs=-1)
 names.append(name)
 results.append(cv_results)
 msg = "%s: %f(+/- %f)" %(name, cv_results.mean() , cv_results.std())
 print(msg)
 fig = plt.figure(figsize=(15,6))
fig.suptitle('Classifier Algorithm Comparison', fontsize=22)
 ax = fig.add_subplot(111)
 sns.boxplot(x=names, y=results)
 ax.set_xticklabels(names)
 ax.set_xlabel("Algorithmn", fontsize=20)
 ax.set_ylabel("Accuracy of Models", fontsize=18)
 ax.set_xticklabels(ax.get_xticklabels() ,rotation=45)
 plt.show()
weather_train_df.isna().sum()<count_missing_values>
perm_xgb = PermutationImportance(XGBClassifier().fit(X_train, y_train), random_state=1 ).fit(X_test,y_test)
 eli5.show_weights(perm_xgb, feature_names = train_ready.drop('Survived',axis=1 ).columns.tolist() )
weather_test_df.isna().sum()<groupby>
perm_knn = PermutationImportance(KNeighborsClassifier(n_neighbors=8 ).fit(X_train, y_train), random_state=1 ).fit(X_test,y_test)
 eli5.show_weights(perm_knn, feature_names = train_ready.drop('Survived',axis=1 ).columns.tolist() )
weather_train_df = weather_train_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))<groupby>
perm_gbc = PermutationImportance(GradientBoostingClassifier(n_estimators=100 ).fit(X_train, y_train), random_state=1 ).fit(X_test,y_test)
 eli5.show_weights(perm_gbc, feature_names = train_ready.drop('Survived',axis=1 ).columns.tolist() )
weather_test_df = weather_test_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))<count_missing_values>
perm_gbc = PermutationImportance(RidgeClassifier().fit(X_train, y_train), random_state=1 ).fit(X_test,y_test)
 eli5.show_weights(perm_gbc, feature_names = train_ready.drop('Survived',axis=1 ).columns.tolist() )
weather_train_df.isna().sum()<count_missing_values>
scaler = StandardScaler()
scaler.fit(X)
 scaled_features = scaler.transform(X)
 train_sc = pd.DataFrame(scaled_features)
 X_csv_test = test_ready.values
scaler.fit(X_csv_test)
 scaled_features_test = scaler.transform(X_csv_test)
 test_sc = pd.DataFrame(scaled_features_test )
weather_test_df.isna().sum()<feature_engineering>
neighbors = np.arange(1, 19)
 train_accuracy = np.empty(len(neighbors))
test_accuracy = np.empty(len(neighbors))
for i, k in enumerate(neighbors):
knn = KNeighborsClassifier(n_neighbors=k)
 knn.fit(X_train, y_train)
 train_accuracy[i] = knn.score(X_train, y_train)
 test_accuracy[i] = knn.score(X_test, y_test)
 plt.title('k-NN: Varying Number of Neighbors')
 plt.plot(neighbors, test_accuracy, label = 'Testing Accuracy')
 plt.plot(neighbors, train_accuracy, label = 'Training Accuracy')
 plt.legend()
plt.xlabel('Number of Neighbors')
 plt.ylabel('Accuracy' )
train_df['meter_reading'] = np.log1p(train_df['meter_reading'] )<categorify>
knn_6 = KNeighborsClassifier(n_neighbors = 6)
 knn_6.fit(scaled_features,y)
 y_pred_knn_6 = knn_6.predict(scaled_features_test )
le = LabelEncoder()
metadata_df['primary_use'] = le.fit_transform(metadata_df['primary_use'] )<categorify>
knn_10 = KNeighborsClassifier(n_neighbors = 10)
 knn_10.fit(scaled_features,y)
 y_pred_knn_10 = knn_10.predict(scaled_features_test )
metadata_df = reduce_mem_usage(metadata_df)
 train_df = reduce_mem_usage(train_df)
 test_df = reduce_mem_usage(test_df)
 weather_train_df = reduce_mem_usage(weather_train_df)
 weather_test_df = reduce_mem_usage(weather_test_df )<train_model>
result_knn_6 = pd.read_csv(".. /input/gender_submission.csv")
 result_knn_6['Survived'] = y_pred_knn_6
result_knn_6.to_csv('Titanic_knn_5.csv', index=False )
print(f'Training data shape: {train_df.shape}')
 print(f'Weather training shape: {weather_train_df.shape}')
 print(f'Weather training shape: {weather_test_df.shape}')
 print(f'Weather testing shape: {metadata_df.shape}')
 print(f'Test data shape: {test_df.shape}' )<merge>
result_knn_10 = pd.read_csv(".. /input/gender_submission.csv")
 result_knn_10['Survived'] = y_pred_knn_10
result_knn_10.to_csv('Titanic_knn_7.csv', index=False )
%%time
full_train_df = train_df.merge(metadata_df, on='building_id', how='left')
 full_train_df = full_train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left' )<filter>
logreg = LogisticRegression()
logreg.fit(scaled_features,y)
 y_pred_logreg = logreg.predict(scaled_features_test)
 y_pred_logreg.shape
full_train_df = full_train_df.loc[~(full_train_df['air_temperature'].isnull() & full_train_df['cloud_coverage'].isnull() & full_train_df['dew_temperature'].isnull() & full_train_df['precip_depth_1_hr'].isnull() & full_train_df['sea_level_pressure'].isnull() & full_train_df['wind_direction'].isnull() & full_train_df['wind_speed'].isnull() & full_train_df['offset'].isnull())]<drop_column>
result_logreg = pd.read_csv(".. /input/gender_submission.csv")
 result_logreg['Survived'] = y_pred_logreg
result_logreg.to_csv('Titanic_logreg.csv', index=False )
del train_df
del weather_train_df
gc.collect()<merge>
clf = xgb.XGBClassifier(n_estimators=250, random_state=4,bagging_fraction= 0.791787170136272, colsample_bytree= 0.7150126733821065,feature_fraction= 0.6929758008695552,gamma= 0.6716290491053838,learning_rate= 0.030240003246947006,max_depth= 2,min_child_samples= 5,num_leaves= 15,reg_alpha= 0.05822089056228967,reg_lambda= 0.14016232510869098,subsample= 0.9)
 clf.fit(scaled_features, y)
 y_pred_xgb= clf.predict(scaled_features_test )
%%time
full_test_df = test_df.merge(metadata_df, on='building_id', how='left')
 full_test_df = full_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left' )<drop_column>
result_xgb = pd.read_csv(".. /input/gender_submission.csv")
 result_xgb['Survived'] = y_pred_xgb
result_xgb.to_csv('Titanic_xgb.csv', index=False )
del metadata_df
del weather_test_df
del test_df
gc.collect()<train_model>
rcf= RidgeClassifier()
rcf.fit(scaled_features, y)
 y_pred_rcf= rcf.predict(scaled_features_test )
print(f'Average meter reading: {full_train_df.meter_reading.mean() } kWh' )<create_dataframe>
result_rcf = pd.read_csv(".. /input/gender_submission.csv")
 result_rcf['Survived'] = y_pred_rcf
result_rcf.to_csv('Titanic_rcf.csv', index=False )
pd.DataFrame(full_train_df.isna().sum().sort_values(ascending=False), columns=['NaN Count'] )<feature_engineering>
gbc= GradientBoostingClassifier(n_estimators=100)
 gbc.fit(scaled_features, y)
 y_pred_gbc= gbc.predict(scaled_features_test )
def mean_without_overflow_fast(col):
col /= len(col)
 return col.mean() * len(col )<sort_values>
result_gbc = pd.read_csv(".. /input/gender_submission.csv")
 result_gbc['Survived'] = y_pred_gbc
result_gbc.to_csv('Titanic_gbc.csv', index=False )