Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<drop_column>
warnings.filterwarnings("ignore")
x_test = test.drop(['Id','groupId','matchId'],axis=1 )<prepare_x_and_y>
df_train = pd.read_csv("/kaggle/input/titanic/train.csv")
 df_test = pd.read_csv("/kaggle/input/titanic/test.csv")
 df_sample_sub = pd.read_csv("/kaggle/input/titanic/gender_submission.csv")
 df_all = df_train.append(df_test, ignore_index=True)
 print("Titanic Dataset Summary:")
 display(df_all.head())
 print("Stats of some features:")
 display(df_all.describe())
 print(df_all.info())
 plt.figure(figsize=(8,5))
sns.heatmap(df_all.isnull())
 plt.show()
m = df_all.shape[0]
print('Number of missing values in',m, 'examples')
 display(df_all.isnull().sum())
X_test = x_test.values<train_model>
df_all['Title'] = df_all.Name.apply(lambda name: name.split(',')[1].split('.')[0].strip())
 display(df_all.Title)
 new_titles = {
"Capt": "Officer",
"Col": "Officer",
"Major": "Officer",
"Jonkheer": "Royalty",
"Don": "Royalty",
"Sir" : "Royalty",
"Dr": "Officer",
"Rev": "Officer",
"the Countess":"Royalty",
"Dona": "Royalty",
"Mme": "Mrs",
"Mlle": "Miss",
"Ms": "Mrs",
"Mr" : "Mr",
"Mrs" : "Mrs",
"Miss" : "Miss",
"Master" : "Master",
"Lady" : "Royalty"
}
df_all.Title = df_all.Title.map(new_titles)
 display(df_all.Title.value_counts())
model = build_model()
model.fit(train_data, train_targets,epochs=70, batch_size=16, verbose=1)
 test_mse_score, test_mae_score = model.evaluate(test_data, test_targets )<predict_on_test>
most_embarked = df_all.Embarked.value_counts().index[0]
df_all.Embarked = df_all.Embarked.fillna(most_embarked)
 df_all.Fare = df_all.Fare.fillna(df_all.Fare.median())
prediction = model.predict(X_test )<prepare_output>
print('Number of missing values in',m, 'examples')
 display(df_all.isnull().sum() )
sample['winPlacePerc'] = prediction<save_to_csv>
df_all.drop('Name', axis =1, inplace=True)
 df_all.drop('Ticket', axis =1, inplace=True)
 df_all.drop('PassengerId', axis=1, inplace = True)
 display(df_all )
sample.to_csv('sample_submission_v1.csv', index=False )<load_from_csv>
Sex = {"male": 0, "female":1}
df_all["Sex"] = df_all.Sex.map(Sex)
 df_all['Partner'] = df_all['SibSp'] + df_all['Parch']
df_all.drop(['SibSp', 'Parch'], axis=1, inplace=True)
 df_all = pd.get_dummies(df_all, columns = ['Title','Embarked'])
 display(df_all.head() )
train_data_df = pd.read_csv('.. /input/train.csv')
 test_data_df = pd.read_csv('.. /input/test.csv' )<groupby>
def logistic_regression(X, y, alpha=1e-3, num_iter=30,random_state=42):
np.random.seed(random_state)
 d, m = X.shape
K = np.max(y)+ 1
w = np.random.randn(d, K)
 def softmax(x):
s = np.exp(x)/ np.sum(np.exp(x))
return s
def one_hot(y, k):
y_one_hot = np.eye(k)[y]
return y_one_hot
def h(x, w):
p = softmax(w.T @ x)
 return p
def cost(pred, y):
c = np.sum(- one_hot(y, K ).T * np.log(pred))
return c
def grad(w, x, y):
Y = one_hot(y, K ).T
b = h(x, w)- Y
b = np.reshape(b,(-1, 1))
x = x.reshape(-1, 1)
 g = x @ np.reshape(b,(-1, 1)).T
return g
for i in range(num_iter):
id = np.random.permutation(m)
 for j in id:
gradient = grad(w, X[:, j], y[j])
 w -= alpha * gradient
return w
mean_group_features = train_data_df.groupby(['matchId','groupId'])[train_data_df.columns[3:-1]].agg('mean' ).reset_index().loc[:, 'assists':'winPoints']
max_group_features = train_data_df.groupby(['matchId','groupId'])[train_data_df.columns[3:-1]].agg('max' ).reset_index().loc[:, 'assists':'winPoints']
min_group_features = train_data_df.groupby(['matchId','groupId'])[train_data_df.columns[3:-1]].agg('min' ).reset_index().loc[:, 'assists':'winPoints']
std_group_features = train_data_df.groupby(['matchId','groupId'])[train_data_df.columns[3:-1]].agg('std' ).reset_index().loc[:, 'assists':'winPoints']<merge>
def ridge_classifier(X, y, lambd=1e-4):
d, m = X.shape
k = np.max(y)+ 1
w = np.linalg.inv(X @ X.T + lambd * np.eye(d)) @ X @ np.eye(k)[y]
return w
features_one = mean_group_features.join(max_group_features, lsuffix='_mean', rsuffix='_max')
 features_two = min_group_features.join(std_group_features, lsuffix='_min', rsuffix='_std')
 features = features_one.join(features_two)
 features = features.fillna(0.0)
 features<groupby>
def error(X, y, w):
m = np.shape(y)
 y_pred = w.T @ X
y_pred = np.argmax(y_pred, axis=0)
 err = np.sum(y_pred == y)/ m
return err
targets = train_data_df.groupby(['matchId', 'groupId'])['winPlacePerc'].agg('mean' ).reset_index() ['winPlacePerc']
targets<define_variables>
mms = MinMaxScaler()
X = df_all.drop('Survived', axis=1 ).iloc[:891].values
y =(df_all["Survived"].iloc[:891].values ).astype(int)
 X = mms.fit_transform(X)
 X_test = df_all.drop('Survived', axis=1 ).iloc[891:].values
X_test = mms.fit_transform(X_test )
<import_modules>
scores_lr = []
scores_ls = []
fold =1
for tr, val in KFold(n_splits=5, random_state=42 ).split(X,y):
X_train = X[tr]
X_val = X[val]
y_train = y[tr]
y_val = y[val]
best_W_LR = logistic_regression(X_train.T, y_train, alpha=1e-3, num_iter=300,random_state=42)
 val_acc_LR = error(X_val.T, y_val, best_W_LR)
 scores_lr.append(val_acc_LR)
 print(f'Validation acc LR: Fold {fold}:', val_acc_LR)
 W_LS = ridge_classifier(X_train.T, y_train, lambd=1e-4)
 val_acc_LS = error(X_val.T, y_val, W_LS)
 scores_ls.append(val_acc_LS)
 print(f'Validation acc LS: Fold {fold}:', val_acc_LS)
 fold +=1
print('-------------------------------')
 print("Accuracy Logistic Regression: %0.2f(+/- %0.2f)" %(np.mean(scores_lr), np.std(scores_lr)* 2))
print("Accuracy Least Squares Ridge: %0.2f(+/- %0.2f)" %(np.mean(scores_ls), np.std(scores_ls)* 2))
import sklearn<normalization>
y_preds_LS =(np.argmax(W_LS.T @ X_test.T, axis=0)).astype(int)
 df_sample_sub.loc[:, 'Survived'] = y_preds_LS
df_sample_sub.to_csv('submission0.csv', index=False)
 display(df_sample_sub.head())
scaler = MinMaxScaler(feature_range=(-1, 1)).fit(features.values)
<choose_model_class>
def test_clfs(clfs):
for clf in clfs:
print('------------------------------------------')
 start = time()
clf = clf(random_state=42)
 scores = cross_val_score(clf, X, y, cv=5)
 print(str(clf), 'results:')
 print("Accuracy: %0.2f(+/- %0.2f)" %(scores.mean() , scores.std() * 2))
end = time()
print('Processing time', end-start,'s')
 models = [RandomForestClassifier, LogisticRegression,XGBClassifier]
test_clfs(models)
bayes_cv_tuner = BayesSearchCV(
 estimator = CatBoostRegressor(iterations = 1500, eval_metric='MAE')
 ,
search_spaces = {
'learning_rate': [0.05, 0.1, 0.15, 0.2, 0.25, 0.3],
'max_depth':(4, 6),
},
scoring = 'neg_mean_absolute_error',
cv = KFold(
 n_splits=5,
shuffle=True,
random_state=42 
),
n_jobs = 1,
n_iter = 6,
verbose = 0,
refit = True,
random_state = 42
)<define_variables>
from sklearn.model_selection import GridSearchCV
%env JOBLIB_TEMP_FOLDER=/tmp<train_model>
clf1 = RandomForestClassifier(max_depth=9, min_samples_leaf=4, min_samples_split=2,
n_estimators=9, random_state=42, n_jobs=-1)
 clf1.fit(X, y)
 y_preds_RF = clf1.predict(X_test ).astype(int)
 df_sample_sub.loc[:, 'Survived'] = y_preds_RF
df_sample_sub.to_csv('submission1.csv', index=False)
 display(df_sample_sub.head())
bayes_cv_tuner.fit(scaler.transform(features.values), targets.values )<find_best_params>
clf2 = LogisticRegression(C=48, class_weight='None', fit_intercept= False, penalty='l2', solver='lbfgs')
 clf2.fit(X, y)
 y_preds_LR = clf2.predict(X_test ).astype(int)
 df_sample_sub.loc[:, 'Survived'] = y_preds_LR
df_sample_sub.to_csv('submission2.csv', index=False)
 display(df_sample_sub.head())
model = bayes_cv_tuner.best_estimator_<groupby>
clf3 = XGBClassifier(booster='gbtree', colsample_bytree= 0.6,
gamma=1, max_depth=5, min_child_weight=1, n_estimators=100, subsample=0.8)
 clf3.fit(X, y)
 y_preds_xgb = clf3.predict(X_test ).astype(int)
 df_sample_sub.loc[:, 'Survived'] = y_preds_xgb
df_sample_sub.to_csv('submission3.csv', index=False)
 display(df_sample_sub.head())
mean_group_features = test_data_df.groupby(['matchId','groupId'])[train_data_df.columns[3:-1]].agg('mean' ).reset_index().loc[:, 'assists':'winPoints']
max_group_features = test_data_df.groupby(['matchId','groupId'])[train_data_df.columns[3:-1]].agg('max' ).reset_index().loc[:, 'assists':'winPoints']
min_group_features = test_data_df.groupby(['matchId','groupId'])[train_data_df.columns[3:-1]].agg('min' ).reset_index().loc[:, 'assists':'winPoints']
std_group_features = test_data_df.groupby(['matchId','groupId'])[train_data_df.columns[3:-1]].agg('std' ).reset_index().loc[:, 'assists':'winPoints']<merge>
def create_model(hid_layers ,dropout_rate, lr):
inp1 = tf.keras.layers.Input(shape =(X.shape[1],))
x1 = tf.keras.layers.BatchNormalization()(inp1)
 for i, units in enumerate(hid_layers):
x1 = tf.keras.layers.Dense(units, activation='relu' )(x1)
 x1 = tf.keras.layers.Dropout(dropout_rate )(x1)
 x1 = tf.keras.layers.Dense(1, activation='sigmoid' )(x1)
 model = tf.keras.models.Model(inputs= inp1, outputs= x1)
 model.compile(optimizer ="adam", loss='binary_crossentropy', metrics='accuracy')
 return model
lr=1e-5
hid_layers = [256, 256]
dr = 0.5
model = create_model(hid_layers, dr, lr)
 tf.keras.utils.plot_model(model, show_shapes = True, show_layer_names= False,
rankdir = 'TB', expand_nested = True)
features_one = mean_group_features.join(max_group_features, lsuffix='_mean', rsuffix='_max')
 features_two = min_group_features.join(std_group_features, lsuffix='_min', rsuffix='_std')
 features = features_one.join(features_two)
 features = features.fillna(0.0)
 features<groupby>
df_sub_copy = df_sample_sub.copy()
df_sub_copy.loc[:, 'Survived'] = 0.0
scores=[]
fold = 0
for tr, val in KFold(n_splits=5, random_state=42 ).split(X,y):
X_train = X[tr]
X_val = X[val]
y_train = y[tr]
y_val = y[val]
rlr = ReduceLROnPlateau(monitor = 'val_loss', factor = 0.2, patience = 3, verbose = 0,
min_delta = 1e-4, min_lr = 1e-6, mode = 'min')
 ckp = ModelCheckpoint(f'bests_weights.hdf5', monitor = 'val_loss', verbose = 0,
save_best_only = True, save_weights_only = True, mode = 'min')
 es = EarlyStopping(monitor = 'val_loss', min_delta = 1e-4, patience = 15, mode = 'min',
baseline = None, restore_best_weights = True, verbose = 0)
 history = model.fit(X_train, y_train, validation_data=(X_val, y_val), callbacks=[rlr,ckp,es],
epochs=300, verbose=0)
 scores.append(np.max(history.history['val_accuracy']))
model.load_weights(f'bests_weights.hdf5')
 y_preds_nn = model.predict(X_test)
 df_sub_copy.loc[:, 'Survived'] += y_preds_nn.reshape(-1)
 print(f'fold',str(fold)+':',scores[fold])
 K.clear_session()
fold+=1
print("Accuracy: %0.2f(+/- %0.2f)" %(np.mean(scores), np.std(scores)* 2))
matches = test_data_df.groupby(['matchId', 'groupId'])['matchId'].agg('mean' ).values<groupby>
df_sample_sub.loc[:, 'Survived'] =(np.round(df_sub_copy.loc[:,'Survived']/ 5)).astype(int)
 display(df_sample_sub.head())
 df_sample_sub.to_csv('submission4.csv', index=False)
<normalization><EOS> 
sub0= pd.read_csv('submission0.csv')
 sub1 = pd.read_csv('submission1.csv')
 sub2 = pd.read_csv('submission2.csv')
 sub3 = pd.read_csv('submission3.csv')
 sub4 = pd.read_csv('submission4.csv')
 sub_vot = np.round(( sub0['Survived']+sub1['Survived']+sub2['Survived']+sub3['Survived']+sub4['Survived'])/5 ).astype(int)
 df_sample_sub.loc[:, 'Survived'] = sub_vot
df_sample_sub.to_csv('submission5.csv', index=False)
 display(df_sample_sub.head() )