Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<drop_column>
warnings.filterwarnings("ignore" )
train.drop(2744604, inplace=True )<filter>
data = pd.read_csv(".. /input/titanic/train.csv")
 print(data.shape)
 data.head()
train[train['winPlacePerc'].isnull() ]<feature_engineering>
data_explore = data.copy()
train['killsNorm'] = train['kills']*(( 100-train['playersJoined'])/100 + 1)
 train['damageDealtNorm'] = train['damageDealt']*(( 100-train['playersJoined'])/100 + 1)
 train['maxPlaceNorm'] = train['maxPlace']*(( 100-train['playersJoined'])/100 + 1)
 train['matchDurationNorm'] = train['matchDuration']*(( 100-train['playersJoined'])/100 + 1)
 to_show = ['Id', 'kills','killsNorm','damageDealt', 'damageDealtNorm', 'maxPlace', 'maxPlaceNorm', 'matchDuration', 'matchDurationNorm']
train[to_show][0:11]<feature_engineering>
data_explore = data_explore.drop(columns=['PassengerId', 'Name', 'Ticket', 'Cabin'], axis=1)
 data_explore.shape
train['healsandboosts'] = train['heals'] + train['boosts']
train[['heals', 'boosts', 'healsandboosts']].tail()<feature_engineering>
max_embarked = data_explore['Embarked'].value_counts().idxmax()
data_explore['Embarked'] = data_explore['Embarked'].fillna(max_embarked )
train['totalDistance'] = train['rideDistance'] + train['walkDistance'] + train['swimDistance']
train['killsWithoutMoving'] =(( train['kills'] > 0)&(train['totalDistance'] == 0))<feature_engineering>
data_explore['Family'] = data_explore['SibSp'] + data_explore['Parch']
data_explore['Family'].loc[data_explore['Family'] > 0] = "Yes"
data_explore['Family'].loc[data_explore['Family'] == 0] = "No"
train['headshot_rate'] = train['headshotKills'] / train['kills']
train['headshot_rate'] = train['headshot_rate'].fillna(0 )<drop_column>
label_encoder = LabelEncoder()
data_explore["sex_enc"] = label_encoder.fit_transform(data_explore["Sex"])
 print(label_encoder.classes_)
 data_explore["embarked_enc"] = label_encoder.fit_transform(data_explore['Embarked'])
 print(label_encoder.classes_)
 data_explore["Family_enc"] = label_encoder.fit_transform(data_explore["Family"])
 print(label_encoder.classes_ )
train.drop(train[train['killsWithoutMoving'] == True].index, inplace=True )<filter>
def fill_with_mean(df, num_cols):

 for col in num_cols:
total_null = df[col].isna().sum()
if total_null>0:
mean = df[col].mean()
std = df[col].std()
lower = mean - std
upper = mean + std
random_values = np.random.randint(lower, upper, total_null)
 df[col][np.isnan(df[col])] = random_values.copy()
return df
train[train['roadKills'] > 10]<drop_column>
data_explore = fill_with_mean(data_explore, ['Age'] )
train.drop(train[train['roadKills'] > 10].index, inplace=True )<drop_column>
data_explore_survived = data_explore[data_explore['Survived']==1]
data_explore_not_survived = data_explore[data_explore['Survived']==0]
train.drop(train[train['kills'] > 30].index, inplace=True )<drop_column>
def get_person_type(passanger):
age, sex = passanger
return 'child' if age < 16 else sex
data_explore["Person"] = data_explore[["Age", "Sex"]].apply(get_person_type, axis=1)
 data_explore_survived = data_explore[data_explore['Survived']==1]
data_explore_not_survived = data_explore[data_explore['Survived']==0]
data_explore["Person_enc"] = label_encoder.fit_transform(data_explore["Person"])
 print(label_encoder.classes_ )
train.drop(train[train['longestKill'] >= 1000].index, inplace=True )<drop_column>
overall_age = dict(data_explore["Person"].value_counts())
 overall_age = sorted(overall_age.items())
 overall_age_values = [item[1] for item in overall_age]
overall_age_label = [item[0] for item in overall_age]
survived_age = dict(data_explore_survived["Person"].value_counts())
 survived_age = sorted(survived_age.items())
 survived_age_values = [item[1] for item in survived_age]
train.drop(train[train['walkDistance'] >= 10000].index, inplace=True )<drop_column>
overall_embarked = dict(data_explore["Embarked"].value_counts())
 overall_embarked = sorted(overall_embarked.items())
 overall_embarked_values = [item[1] for item in overall_embarked]
overall_embarked_labels = [item[0] for item in overall_embarked]
survived_embarked = dict(data_explore_survived["Embarked"].value_counts())
 survived_embarked = sorted(survived_embarked.items())
 survived_embarked_values = [item[1] for item in survived_embarked]
train.drop(train[train['rideDistance'] >= 20000].index, inplace=True )<filter>
overall_pclass = dict(data_explore["Pclass"].value_counts())
 overall_pclass = sorted(overall_pclass.items())
 overall_pclass_labels = ["Upper Class", "Middle Class", "Lower Class"]
overall_pclass_values = [item[1] for item in overall_pclass]
survived_pclass = dict(data_explore_survived["Pclass"].value_counts())
 survived_pclass = sorted(survived_pclass.items())
 survived_pclass_values = [item[1] for item in survived_pclass]
train[train['swimDistance'] >= 2000]<drop_column>
overall_family = dict(data_explore["Family"].value_counts())
 overall_family = sorted(overall_family.items())
 overall_family_labels = ["No", "Yes"]
overall_family_values = [item[1] for item in overall_family]
survived_family = dict(data_explore_survived["Family"].value_counts())
 survived_family = sorted(survived_family.items())
 survived_family_values = [item[1] for item in survived_family]
train.drop(train[train['swimDistance'] >= 2000].index, inplace=True )<drop_column>
corr_matrix['Survived'].sort_values(ascending=False )
train.drop(train[train['weaponsAcquired'] >= 80].index, inplace=True )<drop_column>
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.base import BaseEstimator, TransformerMixin
train.drop(train[train['heals'] >= 40].index, inplace=True )<count_unique_values>
X = data.drop(columns=['Survived'], axis=1)
 y = data['Survived'].copy()
X.shape, X.columns
print('There are {} different Match types in the dataset.'.format(train['matchType'].nunique()))<categorify>
split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)
 for train_index, test_index in split.split(X, y):
strat_train_set = data.iloc[train_index]
strat_test_set = data.iloc[test_index]
X_train = strat_train_set.drop('Survived', axis=1)
 y_train = strat_train_set['Survived'].copy()
X_test = strat_test_set.drop('Survived', axis=1)
 y_test = strat_test_set['Survived'].copy()
X_train.shape, X_test.shape
train = pd.get_dummies(train, columns=['matchType'])
 matchType_encoding = train.filter(regex='matchType')
 matchType_encoding.head()<data_type_conversions>
def fill_with_mean(df, num_cols):
for col in num_cols:
total_null = df[col].isna().sum()
if total_null>0:
mean = df[col].mean()
std = df[col].std()
lower = mean - std
upper = mean + std
random_values = np.random.randint(lower, upper, total_null)
 df[col][np.isnan(df[col])] = random_values.copy()
return df
train['groupId'] = train['groupId'].astype('category')
 train['matchId'] = train['matchId'].astype('category')
 train['groupId_cat'] = train['groupId'].cat.codes
train['matchId_cat'] = train['matchId'].cat.codes
train.drop(columns=['groupId', 'matchId'], inplace=True)
 train[['groupId_cat', 'matchId_cat']].head()<drop_column>
class AddCustomAttribute(BaseEstimator, TransformerMixin):

 def __init__(self):

 self.cat_imp=None
self.cat_encoder=None
self.scaler=None
def fit(self, X, y=None):
return self
def transform(self, X):
try:
X = fill_with_mean(X, ['Age', 'SibSp', 'Parch'])
 if self.cat_imp==None:
self.cat_imp = SimpleImputer(strategy="most_frequent")
 X['Sex'] = self.cat_imp.fit_transform(X[['Sex']])
 else:
X['Sex'] = self.cat_imp.transform(X[['Sex']])
 X['Person'] = X[['Age', 'Sex']].apply(self.get_person_type, axis=1)
 X['Family'] = X['SibSp'] + X['Parch']
X['Family'].loc[X['Family'] > 0] = 1
X['Family'].loc[X['Family'] == 0] = 0
if self.scaler==None:
self.scaler = StandardScaler()
X['Age'] = self.scaler.fit_transform(X[['Age']])
 else:
X['Age'] = self.scaler.transform(X[['Age']])
 if self.cat_encoder==None:
self.cat_encoder = OneHotEncoder(handle_unknown='ignore')
 X_new = self.cat_encoder.fit_transform(X[['Person', 'Family']])
 else:
X_new = self.cat_encoder.transform(X[['Person', 'Family']])
 X_new = pd.DataFrame(X_new.toarray())
 X_new.index = X.index
X = pd.concat([X, X_new], axis=1)
 X = X.drop(columns=['Sex', 'Person', 'SibSp', 'Parch', 'Family'], axis=1)
 except:
print("Error generated in AddCustomAttribute!!!")
 return X
@staticmethod
def get_person_type(passanger):
sex_enc = dict({'child':0, 'female':1, 'male':2})
 age, sex = passanger
return 'child' if age < 16 else sex
train.drop(columns = ['Id'], inplace=True )<prepare_x_and_y>
class FillWithMeanSD(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self, X, y=None):
return self
def transform(self, X):
try:
num_cols = X.columns
for col in num_cols:
total_null_fare = X[col].isna().sum()
if total_null_fare>0:
mean = X[col].mean()
std = X[col].std()
lower = mean - std
upper = mean + std
random_values = np.random.randint(lower, upper, total_null_fare)
 X[col][np.isnan(X[col])] = random_values.copy()
except:
print("Error generated in FillWithSD!!!")
 return X
df = df_sample.drop(columns = ['winPlacePerc'])
 y = df_sample['winPlacePerc']<split>
cat_pipeline = Pipeline([('cat_imputer', SimpleImputer(strategy="most_frequent")) , 
('cat_encoder', OneHotEncoder())])
 num_pipeline = Pipeline([('num_imputer', FillWithMeanSD()), 
('scaler', StandardScaler())])
 pre_process = ColumnTransformer([('drop_attrs', 'drop', ['PassengerId', 'Name', 'Ticket', 'Cabin']), 
('num_pipeline', num_pipeline, ['Pclass', 'Fare']), 
('cat_pipeline', cat_pipeline, ['Embarked']), 
('custom_attr', AddCustomAttribute() , ['Age', 'Sex', 'SibSp', 'Parch'])],
remainder='passthrough' )
def split_vals(a, n : int):
return a[:n].copy() , a[n:].copy()
val_perc = 0.12
n_valid = int(val_perc * sample)
 n_trn = len(df)-n_valid
raw_train, raw_valid = split_vals(df_sample, n_trn)
 X_train, X_valid = split_vals(df, n_trn)
 y_train, y_valid = split_vals(y, n_trn)
 print('Sample train shape: ', X_train.shape,
'Sample target shape: ', y_train.shape,
'Sample validation shape: ', X_valid.shape )<compute_train_metric>
feature_columns = ['Pclass', 'Fare', 'C', 'Q', 'S', 'Age', 'person_child', 'person_female', 'person_male',
'family_no', 'family_yes']
def print_score(m : RandomForestRegressor):
res = ['mae train: ', mean_absolute_error(m.predict(X_train), y_train),
'mae val: ', mean_absolute_error(m.predict(X_valid), y_valid)]
if hasattr(m, 'oob_score_'): res.append(m.oob_score_)
 print(res )<train_model>
kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42 )
m1 = RandomForestRegressor(n_estimators=40, min_samples_leaf=3, max_features='sqrt',
n_jobs=-1)
 m1.fit(X_train, y_train)
 print_score(m1 )<compute_train_metric>
lgr_grid_parm=[{'solver':['liblinear', 'lbfgs'], 'C':list(np.linspace(0.01, 1, 15)) , 'penalty':['l1', 'l2'], 'class_weight':[None, 'balanced']}]
lgr_grid_search = GridSearchCV(LogisticRegression(random_state=42, n_jobs=-1),
lgr_grid_parm, cv=kf, scoring="accuracy", return_train_score=True, n_jobs=-1)
 lgr_grid_search.fit(X_train_transformed, y_train )
fi = rf_feat_importance(m1, df); fi[:10]<features_selection>
lgr_grid_search.best_params_, lgr_grid_search.best_score_
to_keep = fi[fi.imp>0.005].cols
print('Significant features: ', len(to_keep))
to_keep<split>
train_models = []
train_models.append(['Logistic Regression', lgr_grid_search.best_params_, lgr_grid_search.best_score_] )
df_keep = df[to_keep].copy()
X_train, X_valid = split_vals(df_keep, n_trn )<train_model>
best_lgr_clf = lgr_grid_search.best_estimator_
best_lgr_clf
m2 = RandomForestRegressor(n_estimators=80, min_samples_leaf=3, max_features='sqrt',
n_jobs=-1)
 m2.fit(X_train, y_train)
 print_score(m2 )<split>
svc_grid_parm=[{'C':list(np.linspace(0.01, 1, 20)) , 'penalty':['l1', 'l2'], 'class_weight':[None, 'balanced']}]
svc_grid_search = GridSearchCV(LinearSVC(loss="hinge", random_state=42), svc_grid_parm, cv=kf, scoring="accuracy",
return_train_score=True, n_jobs=-1)
 svc_grid_search.fit(X_train_transformed, y_train )
val_perc_full = 0.12
n_valid_full = int(val_perc_full * len(train))
n_trn_full = len(train)-n_valid_full
df_full = train.drop(columns = ['winPlacePerc'])
 y = train['winPlacePerc']
df_full = df_full[to_keep]
X_train, X_valid = split_vals(df_full, n_trn_full)
 y_train, y_valid = split_vals(y, n_trn_full)
 print('Sample train shape: ', X_train.shape,
'Sample target shape: ', y_train.shape,
'Sample validation shape: ', X_valid.shape )<train_model>
svc_grid_search.best_params_, svc_grid_search.best_score_
m3 = RandomForestRegressor(n_estimators=70, min_samples_leaf=3, max_features=0.5,
n_jobs=-1)
 m3.fit(X_train, y_train)
 print_score(m3 )<feature_engineering>
best_svc_clf = svc_grid_search.best_estimator_
best_svc_clf
test['headshot_rate'] = test['headshotKills'] / test['kills']
test['headshot_rate'] = test['headshot_rate'].fillna(0)
 test['totalDistance'] = test['rideDistance'] + test['walkDistance'] + test['swimDistance']
test['playersJoined'] = test.groupby('matchId')['matchId'].transform('count')
 test['killsNorm'] = test['kills']*(( 100-test['playersJoined'])/100 + 1)
 test['damageDealtNorm'] = test['damageDealt']*(( 100-test['playersJoined'])/100 + 1)
 test['maxPlaceNorm'] = test['maxPlace']*(( 100-train['playersJoined'])/100 + 1)
 test['matchDurationNorm'] = test['matchDuration']*(( 100-test['playersJoined'])/100 + 1)
 test['healsandboosts'] = test['heals'] + test['boosts']
test['killsWithoutMoving'] =(( test['kills'] > 0)&(test['totalDistance'] == 0))
test['groupId'] = test['groupId'].astype('category')
 test['matchId'] = test['matchId'].astype('category')
 test['groupId_cat'] = test['groupId'].cat.codes
test['matchId_cat'] = test['matchId'].cat.codes
test_pred = test[to_keep].copy()
test_pred.fillna(0, inplace=True)
 test_pred.head()<save_to_csv>
rf_grid_parm=[{'n_estimators':[50, 100, 200, 300], 'max_depth':[6, 8, 16],
'class_weight':['balanced', 'balanced_subsample', None]}]
rf_grid_search = GridSearchCV(RandomForestClassifier(random_state=42, n_jobs=-1), rf_grid_parm, cv=kf,
scoring="accuracy", return_train_score=True, n_jobs=-1)
 rf_grid_search.fit(X_train_transformed, y_train )
predictions = np.clip(a = m3.predict(test_pred), a_min = 0.0, a_max = 1.0)
 pred_df = pd.DataFrame({'Id' : test['Id'], 'winPlacePerc' : predictions})
 pred_df.to_csv("submission.csv", index=False )<load_from_csv>
rf_grid_search.best_params_, rf_grid_search.best_score_
train_data_df = pd.read_csv('.. /input/train.csv')
 test_data_df = pd.read_csv('.. /input/test.csv' )<normalization>
best_rf_clf = rf_grid_search.best_estimator_
best_rf_clf
train_data_df = reduce_mem_usage(train_data_df)
 test_data_df = reduce_mem_usage(test_data_df )<groupby>
xgb_grid_parm=[{'n_estimators':[50, 100, 200], 'max_depth':[4, 8, 16, 24], 'subsample':[0.5, 0.75, 1.0],
'colsample_bytree':[0.5, 0.75, 1.0], 'gamma':[0, 0.25, 0.5, 0.75, 1.0]}]
xgb_grid_search = GridSearchCV(XGBClassifier(objective='binary:logistic', learning_rate=0.1, random_state=42,
n_jobs=-1), xgb_grid_parm, cv=kf, scoring="accuracy",
return_train_score=True, n_jobs=-1)
 xgb_grid_search.fit(X_train_transformed, y_train )
mean_match_features = train_data_df.groupby(['matchId'])[train_data_df.columns[3:-1]].agg('mean' ).reset_index().loc[:, 'assists':'winPoints']
size_match_features = pd.DataFrame(train_data_df.groupby(['matchId'])[train_data_df.columns[3]].agg('size' ).reset_index() [train_data_df.columns[3]])
 size_match_features.columns = ['match_size'] 
<groupby>
xgb_grid_search.best_params_, xgb_grid_search.best_score_
mean_group_features = train_data_df.groupby(['matchId','groupId'])[train_data_df.columns[3:-1]].agg('mean' ).reset_index().loc[:, 'assists':'winPoints']
max_group_features = train_data_df.groupby(['matchId','groupId'])[train_data_df.columns[3:-1]].agg('max' ).reset_index().loc[:, 'assists':'winPoints']
min_group_features = train_data_df.groupby(['matchId','groupId'])[train_data_df.columns[3:-1]].agg('min' ).reset_index().loc[:, 'assists':'winPoints']
size_group_features = pd.DataFrame(train_data_df.groupby(['matchId','groupId'])[train_data_df.columns[3]].agg('size' ).reset_index() [train_data_df.columns[3]])
 size_group_features.columns = ['group_size']<merge>
best_xgb_clf = xgb_grid_search.best_estimator_
best_xgb_clf
features_three = mean_group_features.join(max_group_features, lsuffix='_group_mean', rsuffix='_group_max')
 features_four = min_group_features.join(size_group_features, lsuffix='_group_min', rsuffix='_group_size')
 features_2 = features_three.join(features_four)
 features_2['matchId'] = train_data_df.groupby(['matchId', 'groupId'])['matchId'].agg('mean' ).values
features_2 = features_2.fillna(0.0)
 features_2 = features_2.replace([np.inf, -np.inf], 0.0 )<merge>
named_estimators = [('logistic', best_lgr_clf),('linear_svc', best_svc_clf), 
('forest', best_rf_clf),('xgb', best_xgb_clf)]
features = pd.merge(features_2, features_1, on='matchId', how='right' )<drop_column>
stack_clf = StackingClassifier(estimators=named_estimators, cv=kf, passthrough=False, n_jobs=-1)
 stack_clf.fit(X_train_transformed, y_train )
features = features.drop(['matchId'], axis=1)
 features<groupby>
stack_acc = cross_val_score(stack_clf, X_train_transformed, y_train, scoring="accuracy", cv=kf, n_jobs=-1)
 stack_acc = np.round(np.median(stack_acc), 4)
 train_models.append(['Stacking Classifier', '', stack_acc] )
targets = train_data_df.groupby(['matchId', 'groupId'])['winPlacePerc'].agg('mean' ).reset_index() ['winPlacePerc']
targets<groupby>
voting_clf = VotingClassifier(estimators=named_estimators, n_jobs=-1)
 voting_clf.fit(X_train_transformed, y_train )
train_data_df.groupby(['matchId', 'groupId'])['winPlacePerc'].agg('max' )<define_variables>
voting_acc = cross_val_score(voting_clf, X_train_transformed, y_train, scoring="accuracy", cv=kf, n_jobs=-1)
 voting_acc = np.round(np.mean(voting_acc), 4)
 train_models.append(['Voting Classifier', '', voting_acc] )
train_features = features.values[0:np.int32(0.8*len(features)) ]
train_targets = targets.values[0:np.int32(0.8*len(features)) ]
val_features = features.values[np.int32(0.8*len(features)) :len(features)]
val_targets = targets.values[np.int32(0.8*len(features)) :len(features)]<import_modules>
pd.set_option('display.max_colwidth', -1)
 train_models_df = pd.DataFrame(train_models, columns=['Model', 'Best Paramas', 'Accuracy'])
 train_models_df
import sklearn<normalization>
results = dict()
best_models = [best_lgr_clf, best_svc_clf, best_rf_clf, best_xgb_clf, stack_clf, voting_clf]
model_names = []
model_accuracy = []
for model in best_models:
test_accuracy_scores = cross_val_score(model, X_test_transformed, y_test, scoring="accuracy", cv=kf, n_jobs=-1)
 test_accuracy_scores = np.round(test_accuracy_scores,4)
 test_accuracy = np.round(np.mean(test_accuracy_scores),4)
 model_names.append(model.__class__.__name__)
 model_accuracy.append(test_accuracy )
scaler = MinMaxScaler(feature_range=(-1, 1)).fit(features.values)
 train_features = scaler.transform(train_features)
 val_features = scaler.transform(val_features )<train_model>
best_model = best_models[np.argmax(model_accuracy)]
best_model
model = CatBoostRegressor(iterations=250, learning_rate=0.1, eval_metric='MAE', max_depth=8)
 model.fit(train_features, train_targets, eval_set=(val_features, val_targets))<groupby>
final_model = Pipeline([('pre_process', pre_process), 
('best_model', best_model)])
 final_model.fit(X_train, y_train )
mean_match_features = test_data_df.groupby(['matchId'])[train_data_df.columns[3:-1]].agg('mean' ).reset_index().loc[:, 'assists':'winPoints']
size_match_features = pd.DataFrame(test_data_df.groupby(['matchId'])[train_data_df.columns[3]].agg('size' ).reset_index() [train_data_df.columns[3]])
 size_match_features.columns = ['match_size'] 
<groupby>
test_data = pd.read_csv(".. /input/titanic/test.csv")
 test_data.info()
mean_group_features = test_data_df.groupby(['matchId','groupId'])[train_data_df.columns[3:-1]].agg('mean' ).reset_index().loc[:, 'assists':'winPoints']
max_group_features = test_data_df.groupby(['matchId','groupId'])[train_data_df.columns[3:-1]].agg('max' ).reset_index().loc[:, 'assists':'winPoints']
min_group_features = test_data_df.groupby(['matchId','groupId'])[train_data_df.columns[3:-1]].agg('min' ).reset_index().loc[:, 'assists':'winPoints']
size_group_features = pd.DataFrame(test_data_df.groupby(['matchId','groupId'])[train_data_df.columns[3]].agg('size' ).reset_index() [train_data_df.columns[3]])
 size_group_features.columns = ['group_size']<merge>
predictions = final_model.predict(test_data )
features_three = mean_group_features.join(max_group_features, lsuffix='_group_mean', rsuffix='_group_max')
 features_four = min_group_features.join(size_group_features, lsuffix='_group_min', rsuffix='_group_size')
 features_2 = features_three.join(features_four)
 features_2['matchId'] = test_data_df.groupby(['matchId', 'groupId'])['matchId'].agg('mean' ).values
features_2 = features_2.fillna(0.0)
 features_2 = features_2.replace([np.inf, -np.inf], 0.0 )<merge>
test_predictions = pd.DataFrame(test_data['PassengerId'])
 test_predictions['Survived'] = predictions.copy()
test_predictions.head()
<groupby><EOS> 
test_predictions.to_csv("./submission.csv", index=False )