Kaggle competition: Titanic - Machine Learning from Disaster
ccc = pop_cols + ext_cols
df_pop[df_pop[ccc].isnull().any(axis=1)][['Country/Region', 'Province/State'] + ccc].drop_duplicates(subset=['Country/Region', 'Province/State'])
<feature_engineering>
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 train_data.head()
df_pop['ExposedDensity'] =(df_pop['Population'] - np.power(df_pop['ConfirmedCases'], 1.43)) /df_pop['Land Area']
density = df_pop.groupby(['Country/Region', 'Province/State'])['Density']
df_pop['PreviousDay-0ExposedDensity'] = df_pop.groupby(['Country/Region', 'Province/State'])['ExposedDensity'].shift(periods=1, fill_value=np.nan)
 df_pop['PreviousDay-0ExposedDensity'] = df_pop.apply(
 lambda row: row['Density'] if np.isnan(row['PreviousDay-0ExposedDensity'])else row['PreviousDay-0ExposedDensity'],
axis=1 
)
 df_pop<train_model>
test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 test_data.head()
model_x_columns_without_dummies = add_cols + pop_cols + ext_cols + previous_days_cols + special_cols2
model_x_columns = model_x_columns_without_dummies + continent_columns
def train_test_split(X, y, test_size=0.3, random_state=0):
day_first = min(X['DayNum'])
 day_last = max(X['DayNum'])
 number_of_days_for_train = int(round(( day_last-day_first+1)*(1-test_size),0))
last_day_for_training = number_of_days_for_train + day_first - 1
X_tr = X[X['DayNum']<=last_day_for_training].copy()
y_tr = y[X['DayNum']<=last_day_for_training].copy()
X_te = X[X['DayNum']>last_day_for_training].copy()
y_te = y[X['DayNum']>last_day_for_training].copy()
return X_tr, X_te, y_tr, y_te
def rmsle(ytrue, ypred):
return np.sqrt(mean_squared_log_error(ytrue, ypred))
def mae(ytrue, ypred):
return mean_absolute_error(ytrue, ypred)
 def mse(ytrue, ypred):
return mean_squarred_error(ytrue, ypred)
 def analyse3(tr_true, tr_pred, val_true, val_pred, test_true, test_pred):
chart_data0 = pd.DataFrame({
'x0': tr_true.flatten() ,
'x1': tr_pred.flatten() ,
'y': tr_true.flatten() -tr_pred.flatten() })
 chart_data1 = pd.DataFrame({
'x0': val_true.flatten() ,
'x1': val_pred.flatten() ,
'y': val_true.flatten() -val_pred.flatten() })
 chart_data2 = pd.DataFrame({
'x0': test_true.flatten() ,
'x1': test_pred.flatten() ,
'y': test_true.flatten() -test_pred.flatten() })
 fig, ax =plt.subplots(1,3)
 sns.scatterplot(x='x0', y='y', data=chart_data0, color='black', ax=ax[0])
 sns.scatterplot(x='x1', y='y', data=chart_data0, color='red', ax=ax[0])
 sns.scatterplot(x='x0', y='y', data=chart_data1, color='black', ax=ax[1])
 sns.scatterplot(x='x1', y='y', data=chart_data1, color='red', ax=ax[1])
 sns.scatterplot(x='x0', y='y', data=chart_data2, color='black', ax=ax[2])
 sns.scatterplot(x='x1', y='y', data=chart_data2, color='red', ax=ax[2])
 print('MAE train: {}'.format(round(rmsle(tr_true, tr_pred), 6)))
 print('MAE val: {}'.format(round(rmsle(val_true, val_pred), 6)))
 print('MAE test: {}'.format(round(rmsle(test_true, test_pred), 6)))
 def prepare_data(df, what_to_predict, test_size=0.3, dropna=False):
df_tmp = df.copy()
if dropna:
df_tmp.dropna(inplace=True)
 df_tmp.loc[df_tmp[what_to_predict]<0, what_to_predict] = 0
data_X = df_tmp[model_x_columns+['Country/Region']]
data_y = df_tmp[what_to_predict].values.flatten()
return train_test_split(data_X, data_y, test_size=test_size, random_state=42)
 def predict_output(input_data, model):
y_pred = np.abs(model.predict(input_data))
return y_pred
def expm1_relu(y):
tmp = np.expm1(y)
 tmp[tmp<0]=0
return np.around(tmp )<normalization>
train_data['IsMale'] = train_data['Sex'].apply(lambda x: 1 if x == 'male' else 0)
 train_data.head()
scaler0 = None
scaler1 = None
scaler2 = None
def scale_data(data):
global scaler0, scaler1, scaler2
data_bis = data.copy()
daynum = data_bis['DayNum'].copy()
memory = dict()
for c in special_cols2:
memory[c] = data_bis[c].copy()
if scaler1:
data_bis[model_x_columns_without_dummies] = scaler0.transform(data[model_x_columns_without_dummies])
 data_bis[model_x_columns_without_dummies] = scaler1.transform(data[model_x_columns_without_dummies])
 data_bis[model_x_columns_without_dummies] = scaler2.transform(data_bis[model_x_columns_without_dummies])
 else:
scaler0 = PowerTransformer()
scaler1 = MinMaxScaler()
scaler2 = StandardScaler()
data_bis[model_x_columns_without_dummies] = scaler0.fit_transform(data[model_x_columns_without_dummies])
 data_bis[model_x_columns_without_dummies] = scaler1.fit_transform(data[model_x_columns_without_dummies])
 data_bis[model_x_columns_without_dummies] = scaler2.fit_transform(data_bis[model_x_columns_without_dummies])
 for c in memory.keys() :
data_bis[c] = memory[c]
data_bis['DayNum'] = daynum
return data_bis
df_pop_bis = scale_data(df_pop)
 df_pop_bis 
<define_variables>
test_data['IsMale'] = test_data['Sex'].apply(lambda x: 1 if x == 'male' else 0)
 test_data.head()
model_path = join('.')
 model_file_f = join(model_path, 'nn_model_f.h5')
 model_file_cc = join(model_path, 'nn_model_cc.h5' )<import_modules>
y = train_data["Survived"]
features = ["Pclass", "IsMale", "SibSp", "Parch", "Age", "Fare"]
X = train_data[features]
X.tail()
class Swish(Activation):
def __init__(self, activation, **kwargs):
super(Swish, self ).__init__(activation, **kwargs)
 self.__name__ = 'swish'
def swish(x, beta = 0.6):
return(x * sigmoid(beta * x))
get_custom_objects().update({'swish': Swish(swish)} )<choose_model_class>
X_filled = KNN(k=3 ).fit_transform(X)
 X_filled[-5:]
model_f = Sequential()
model_f.add(Dense(50, input_dim=len(model_x_columns)-1, activation='swish'))
model_f.add(Dropout(0.2))
model_f.add(Dense(15, activation='elu'))
model_f.add(Dropout(0.2))
model_f.add(Dense(1, activation='elu'))
opt_f = Adam(learning_rate=0.001, beta_1=0.94, beta_2=0.99, amsgrad=False)
 model_f.compile(loss='mean_absolute_error', optimizer=opt_f, metrics=['mae'])
 save_best_only_callback_f = ModelCheckpoint(
 filepath=model_file_f,
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='min',
period=1 
)
 data_X_tr, data_X_rest, data_y_tr, data_y_rest = prepare_data(df_pop_bis, 'FatalitiesDelta', test_size=0.3, dropna=False)
 data_X_val, data_X_test, data_y_val, data_y_test = train_test_split(data_X_rest, data_y_rest, test_size=0.5, random_state=111)
 history = model_f.fit(x=data_X_tr[model_x_columns].drop(columns=['DayNum']), y=data_y_tr,
validation_data=(data_X_val[model_x_columns].drop(columns=['DayNum']), data_y_val),
epochs=170, batch_size=128, verbose=1, callbacks=[save_best_only_callback_f])
 plt.plot(history.history['loss'][0:])
 plt.plot(history.history['val_loss'][0:])
 plt.title('model loss')
 plt.ylabel('loss')
 plt.xlabel('epoch')
 plt.legend(['train', 'val'], loc='upper left')
 plt.show() 
<predict_on_test>
X_filled = pd.DataFrame(data=X_filled,columns=['Pclass','IsMale','SibSp','Parch','Age','Fare'])
 X_filled.head()
model_f = load_model(model_file_f)
 tr_pred = predict_output(data_X_tr[model_x_columns].drop(columns=['DayNum']), model_f)
 val_pred = predict_output(data_X_val[model_x_columns].drop(columns=['DayNum']), model_f)
 test_pred = predict_output(data_X_test[model_x_columns].drop(columns=['DayNum']), model_f)
 analyse3(data_y_tr, tr_pred,
data_y_val, val_pred,
data_y_test, test_pred )<choose_model_class>
X_train, X_valid, y_train, y_valid = train_test_split(X_filled, y, random_state=1)
 model = XGBRFClassifier(n_estimators=1000, max_depth=5, random_state=1, learning_rate=0.5)
 model.fit(X_train, y_train,
early_stopping_rounds=5,
eval_set=[(X_valid, y_valid)],
verbose=False)
 predictions = model.predict(X_valid)
 print("Mean Absolute Error: " + str(mean_absolute_error(predictions, y_valid)) )
model_cc = Sequential()
model_cc.add(Dense(28, input_dim=len(model_x_columns)-1, activation='swish'))
model_cc.add(Dropout(0.0))
model_cc.add(Dense(15, activation='elu'))
model_cc.add(Dropout(0.0))
model_cc.add(Dense(1, activation='swish'))
opt_cc = Adam(learning_rate=0.0001, beta_1=0.988, beta_2=0.99, amsgrad=False)
 model_cc.compile(loss='mean_absolute_error', optimizer=opt_cc, metrics=['mae'])
 save_best_only_callback_cc = ModelCheckpoint(
 filepath=model_file_cc,
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='min',
period=1 
)
 data_X_tr, data_X_rest, data_y_tr, data_y_rest = prepare_data(df_pop_bis, 'ConfirmedCasesDelta', test_size=0.3, dropna=False)
 data_X_val, data_X_test, data_y_val, data_y_test = train_test_split(data_X_rest, data_y_rest, test_size=0.5, random_state=111)
 history = model_cc.fit(x=data_X_tr[model_x_columns].drop(columns=['DayNum']), y=data_y_tr,
validation_data=(data_X_val[model_x_columns].drop(columns=['DayNum']), data_y_val),
epochs=770, batch_size=128, verbose=1, callbacks=[save_best_only_callback_cc])
 plt.plot(history.history['loss'][0:])
 plt.plot(history.history['val_loss'][0:])
 plt.title('model loss')
 plt.ylabel('loss')
 plt.xlabel('epoch')
 plt.legend(['train', 'val'], loc='upper left')
 plt.show()<predict_on_test>
X_test = test_data[features]
X_test_filled = KNN(k=3 ).fit_transform(X_test)
 X_test_filled = pd.DataFrame(data=X_test_filled,columns=['Pclass','IsMale','SibSp','Parch','Age','Fare'])
 X_test_filled
model_cc = load_model(model_file_cc)
 tr_pred = predict_output(data_X_tr[model_x_columns].drop(columns=['DayNum']), model_cc)
 val_pred = predict_output(data_X_val[model_x_columns].drop(columns=['DayNum']), model_cc)
 test_pred = predict_output(data_X_test[model_x_columns].drop(columns=['DayNum']), model_cc)
 analyse3(( data_y_tr),(tr_pred), 
(data_y_val),(val_pred), 
(data_y_test),(test_pred))<feature_engineering>
predictions_final = model.predict(X_test_filled)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions_final})
 output
<merge><EOS> 
output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )