Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<save_to_csv>
%matplotlib inline
y_test = y_test.reshape(( -1, 1))
pred_test_y =(y_test>np.mean(bestscore)).astype(int)
 sub['prediction'] = pred_test_y
sub.to_csv("submission.csv", index=False )<set_options>
train_data = pd.read_csv('.. /input/train.csv')
 test_data = pd.read_csv('.. /input/test.csv')
 full_data = train_data.append(test_data, ignore_index=True, sort=False)
 full_data.head()
warnings.filterwarnings("ignore" )<load_from_csv>
full_data[['Pclass', 'Survived']].groupby(['Pclass'], as_index=True ).mean()
train = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 submission = pd.read_csv(".. /input/covid19-global-forecasting-week-4/submission.csv" )<data_type_conversions>
pclass = pd.get_dummies(data=full_data.Pclass, prefix='Pclass', drop_first=False)
 pclass.head()
train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'] )<feature_engineering>
full_data[['Sex', 'Survived']].groupby(['Sex'], as_index=True ).mean()
train = train[train['Date']<='2020-04-14']
train['part'] = 'train'
test['part'] = 'test'<data_type_conversions>
sex = pd.DataFrame(data=full_data.Sex.map({'female': '1', 'male': '0'} ).astype('int'), columns=['Sex'])
 sex.head()
train['Location'] = train['Province_State'].astype(str)+ train['Country_Region'].astype(str)
 test['Location'] = test['Province_State'].astype(str)+ test['Country_Region'].astype(str )<merge>
full_data[['Embarked', 'Survived']].groupby(['Embarked'], as_index=True ).mean()
test = test.merge(train[['ConfirmedCases','Fatalities','Location','Date']],how='left',on=['Location','Date'] )<filter>
full_data.loc[full_data.Embarked.isnull() , :]
train = train[train['Date']<='2020-04-01']<concatenate>
embarked = pd.get_dummies(data=full_data.Embarked.fillna('C'), prefix='Embarked')
 embarked.head()
data = pd.concat([train,test],axis=0)
 data = data.sort_values(['Country_Region','Date'] )<categorify>
full_data['Title'] = full_data.Name.str.extract('([A-Za-z]+)\.', expand=False)
 full_data[['Title', 'Sex', 'Survived']].groupby(['Sex','Title'], as_index=True ).mean()
data = data.melt(id_vars=['ForecastId','Date','Location','part'],value_vars=['ConfirmedCases','Fatalities'],value_name='Target' ).sort_values(['Location','Date'] )<feature_engineering>
Title_Dictionary = {"Capt": "Mr",
"Col": "Officer",
"Major": "Officer",
"Jonkheer": "Mr",
"Don": "Mr",
"Sir" : "Royalty",
"Dr": "Officer",
"Rev": "Mr",
"Countess": "Royalty",
"Dona": "Mrs",
"Mme": "Mrs",
"Mlle": "Miss",
"Ms": "Mrs",
"Mr" : "Mr",
"Mrs" : "Mrs",
"Miss" : "Miss",
"Master" : "Officer",
"Lady" : "Royalty"}
full_data['Title'] = full_data.Title.map(Title_Dictionary)
 full_data[['Title', 'Sex', 'Survived']].groupby(['Sex', 'Title'], as_index=True ).mean()
data['Day'] = data['Date'].astype(str ).apply(lambda x: int(''.join(x.split('-')[1:])))
 data['Month'] = data.Date.dt.month<categorify>
title = pd.get_dummies(full_data.Title)
 title.head()
data['lag_1'] = data.groupby(['Location','variable'])['Target'].transform(lambda x: x.shift(1))
data['lag_2'] = data.groupby(['Location','variable'])['Target'].transform(lambda x: x.shift(2))
data['lag_3'] = data.groupby(['Location','variable'])['Target'].transform(lambda x: x.shift(3))
data['lag_4'] = data.groupby(['Location','variable'])['Target'].transform(lambda x: x.shift(4))<feature_engineering>
full_data[full_data.Fare.isnull() ]
data['Diff1'] = data['lag_1'] - data['lag_2']
data['Diff2'] = data['lag_2'] - data['lag_3']
data['Diff3'] = data['lag_3'] - data['lag_4']
data['Diffavg'] =(data['Diff1'] + data['Diff2'] +data['Diff3'])/3<feature_engineering>
test_data[["Pclass", "Fare", "Embarked"]].groupby(["Pclass", "Embarked"] ).mean()
data["Inc1"] =(data['Diff1'] / data['lag_2'])*100
data["Inc2"] =(data['Diff2'] / data['lag_3'])*100
data["Inc3"] =(data['Diff3'] / data['lag_4'])*100
data['Incavg'] =(data['Inc1'] + data['Inc2'] +data['Inc3'])/3<drop_column>
fare = pd.DataFrame(data=full_data.Fare.fillna(13.9))
mu_fare = fare.Fare.mean()
sigma_fare =(((fare.Fare-mu_fare)**2 ).mean())**0.5
fare.Fare =(fare.Fare - mu_fare)/ sigma_fare
data = data[data['Date']>'2020-02-19']
data.drop(['Diff1','Diff2','Diff3'],axis=1,inplace=True )<categorify>
age = pd.DataFrame(data=grouped.Age.apply(lambda x: x.fillna(x.median())))
 mu_a = age.Age.mean()
sigma_a =(((age.Age-mu_a)**2 ).mean())**0.5
age.Age =(age.Age - mu_a)/ sigma_a
encoderloc = LabelEncoder()
encodervar = LabelEncoder()
data['Location'] = encoderloc.fit_transform(data['Location'])
 data['variable'] = encodervar.fit_transform(data['variable'])
 data.head()<define_variables>
cabin = pd.DataFrame()
cabin['Cabin'] = full_data.Cabin.fillna('U')
 cabin['Cabin'] = cabin.Cabin.map(lambda x: x[0])
 cabin = pd.get_dummies(cabin, drop_first=False)
 cabin.head()
features = ['Day','Location','variable','lag_1','lag_2','Diffavg','Inc1','Inc2','Incavg','Month']<prepare_x_and_y>
family = pd.DataFrame()
family['FamilySize'] = full_data['Parch'] + full_data['SibSp'] + 1
family['Family_Single'] = family['FamilySize'].map(lambda s: 1 if s == 1 else 0)
 family['Family_Small'] = family['FamilySize'].map(lambda s: 1 if 2 <= s <= 4 else 0)
 family['Family_Large'] = family['FamilySize'].map(lambda s: 1 if 5 <= s else 0)
 mu_size = family.FamilySize.mean()
sigma_size =(((family.FamilySize-mu_size)**2 ).mean())**0.5
family.FamilySize =(family.FamilySize - mu_size)/ sigma_size
family.head()
x_train = data[data['Date']<='2020-04-14']
y_train = x_train['Target']
x_val = data[(data['Date']>='2020-04-02')&(data['Date']<='2020-04-14')]
y_val = x_val['Target']
test_ = data[data['part']=='test'] 
<import_modules>
features = pd.concat([pclass, sex, embarked, title, fare, age, cabin, family] , axis=1)
 features.head()
from xgboost import DMatrix,train,plot_importance,XGBRegressor<init_hyperparams>
labels = train_data.Survived
train_features, val_features, train_labels, val_labels = train_test_split(features[:891],
labels,
test_size = 0.2)
 test_features = features[891:]
train_features.shape, train_labels.shape, val_features.shape, val_labels.shape, test_features.shape
params = {'objective': 'reg:squarederror',
'n_jobs': -1,
'seed': 236,
} 
<import_modules>
m_train = train_features.shape[0]
m_val = val_features.shape[0]
m_test = test_features.shape[0]
n = train_features.shape[1]
print(f" m_train = {m_train} / m_val = {m_val} / m_test = {m_test}, n = {n}" )
from tqdm import tqdm<feature_engineering>
layer_dims = [n, 7, 7, 1]
def feature(test_,ctr):
if ctr==1:
lags = [1]
elif ctr==2:
lags = [1,2]
elif ctr==3:
lags = [1,2,3]
else:
lags = [1,2,3,4]
for i in lags:
test_['lag_'+str(i)] = test_.groupby(['Location','variable'])['Target'].transform(lambda x: x.shift(i))
test_['Diff1'] = test_['lag_1'] - test_['lag_2']
test_['Diff2'] = test_['lag_2'] - test_['lag_3']
test_['Diff3'] = test_['lag_3'] - test_['lag_4']
test_['Diffavg'] =(test_['Diff1'] + test_['Diff2'] +test_['Diff3'])/3
test_["Inc1"] =(test_['Diff1'] / test_['lag_2'])*100
test_["Inc2"] =(test_['Diff2'] / test_['lag_3'])*100
test_["Inc3"] =(test_['Diff3'] / test_['lag_4'])*100
test_['Incavg'] =(test_['Inc1'] + test_['Inc2'] +test_['Inc3'])/3
test_.drop(['Diff1','Diff2','Diff3'],axis=1,inplace=True)
 ctr+=1
return test_<predict_on_test>
k_model = Sequential()
k_model.add(Dense(layer_dims[1], activation='relu',
kernel_regularizer=l2(0.01),
input_dim=n))
k_model.add(Dense(layer_dims[2], activation='relu',
kernel_regularizer=l2(0.01)))
 k_model.add(Dense(layer_dims[3], activation='sigmoid',
kernel_regularizer=l2(0.01)))
 k_model.compile(optimizer=Adam(lr=1e-1),
loss='binary_crossentropy',
metrics=['accuracy'])
 lr_decay = ReduceLROnPlateau(monitor='loss',
patience=1,
verbose=0,
factor=0.5,
min_lr=1e-7)
 k_model.fit(train_features, train_labels, epochs=300,
callbacks=[lr_decay], verbose=0)
 k_train_loss, k_train_acc = k_model.evaluate(train_features, train_labels)
 k_val_loss, k_val_acc = k_model.evaluate(val_features, val_labels)
 print(f'k_model: train accuracy = {round(k_train_acc * 100, 4)}%')
 print(f'k_model: val accuracy = {round(k_val_acc * 100, 4)}%')
 print(f'k_model: val error = {round(( 1 - k_val_acc)* m_val)} examples' )
train_set = DMatrix(x_train[features],y_train)
 val_set = DMatrix(x_val[features],y_val)
 model = train(params,train_set,num_boost_round=100,evals=[(val_set,'validation')],verbose_eval=50)
 ctr = 1
for j in days:
test_set = DMatrix(test_[test_['Day']==j][features])
 test_.loc[test_['Day']==j,'Target'] = model.predict(test_set)
 test_ = feature(test_,ctr)
 ctr+=1<prepare_output>
X_train = train_features.T.values
Y_train = train_labels.T.values.reshape(1, train_labels.shape[0])
 X_val = val_features.T.values
Y_val = val_labels.T.values.reshape(1, val_labels.shape[0])
 X_test = test_features.T.values
X_train.shape, Y_train.shape, X_val.shape, Y_val.shape, X_test.shape
sub = test_[['ForecastId','variable','Target']]<create_dataframe>
def random_mini_batches(X, Y, batch_size):

 m_train = X.shape[1]
mini_batches = []
permutation = list(np.random.permutation(m_train))
shuffled_X = X[:, permutation]
shuffled_Y = Y[:, permutation].reshape(( 1,m_train))
num_complete_minibatches = m_train // batch_size
for k in range(0, num_complete_minibatches):
mini_batch_X = shuffled_X[:, k*batch_size:(k+1)*batch_size]
mini_batch_Y = shuffled_Y[:, k*batch_size:(k+1)*batch_size]
mini_batch =(mini_batch_X, mini_batch_Y)
 mini_batches.append(mini_batch)
 if m_train % batch_size != 0:
mini_batch_X = shuffled_X[:, num_complete_minibatches*batch_size:]
mini_batch_Y = shuffled_Y[:, num_complete_minibatches*batch_size:]
mini_batch =(mini_batch_X, mini_batch_Y)
 mini_batches.append(mini_batch)
 return mini_batches
sub = pd.pivot(sub,index='ForecastId',columns='variable',values='Target' ).reset_index()<data_type_conversions>
class Custom_model(object):
def __init__(self, layer_dims):

 self.layer_dims = layer_dims
self.num_layers = len(layer_dims)
 self.parameters = {}
for l in range(1, len(layer_dims)) :
self.parameters[f"W{l}"] = np.random.randn(layer_dims[l], layer_dims[l-1])*0.01
self.parameters[f"b{l}"] = np.zeros(( layer_dims[l], 1))
def get_layer_dims(self):
return self.layer_dims
def get_num_layers(self):
return self.num_layers
def get_params(self, key):
return self.parameters.get(key)
 def set_params(self, key, value):
self.parameters[key] = value
def forward_propagation(self, X):

 caches = []
L = self.get_num_layers() -1
A = X
for l in range(1, L):
A_prev = A
W, b = self.get_params(f'W{l}'), self.get_params(f'b{l}')
 Z = W.dot(A_prev)+ b
A = np.maximum(0,Z)
 assert(A.shape == Z.shape)
 cache =(A_prev, Z)
 caches.append(cache)
 W, b = self.get_params(f'W{L}'), self.get_params(f'b{L}')
 Z = W.dot(A)+ b
AL = 1 /(1 + np.exp(-Z))
assert(AL.shape ==(1,X.shape[1]))
cache =(A, Z)
 caches.append(cache)
 return AL, caches
def compute_cost(self, AL, Y, lambd):

 m = AL.shape[1]
L = self.get_num_layers() -1
assert(Y.shape == AL.shape)
 cross_entropy_cost =(1./m)*(-Y.dot(np.log(AL ).T)-(1-Y ).dot(np.log(1-AL ).T))
L2_reg_cost = 0
if lambd == 0:
pass
else:
for l in range(1, L+1):
L2_reg_cost +=(1./m)*(lambd/2)*(np.sum(np.square(self.get_params(f'W{l}'))))
cost = cross_entropy_cost + L2_reg_cost
cost = np.squeeze(cost)
 return cost
def backward_propagation(self, AL, Y, caches, lambd):

 L = self.get_num_layers() -1
m = AL.shape[1]
assert(Y.shape == AL.shape)
 grads = {}
dAL = -(np.divide(Y, AL)- np.divide(1-Y, 1-AL))
A_prev, Z = caches[L-1]
s = 1./(1.+ np.exp(-Z))
dZ = dAL * s *(1-s)
 W = self.get_params(f"W{L}")
 dW =(1./m)* dZ.dot(A_prev.T)+(lambd/m)* W
db =(1./m)* np.sum(dZ, axis=1, keepdims=True)
 dA_prev = np.dot(W.T,dZ)
 grads[f"dA{L-1}"] = dA_prev
grads[f"dW{L}"], grads[f"db{L}"] = dW, db
for l in reversed(range(1, L)) :
A_prev, Z = caches[l-1]
dZ = np.array(dA_prev, copy=True)
 dZ[Z <= 0] = 0
W = self.get_params(f"W{l}")
 dW =(1./m)* dZ.dot(A_prev.T)+(lambd/m)* W
db =(1./m)* np.sum(dZ, axis=1, keepdims=True)
 dA_prev = np.dot(W.T,dZ)
 grads[f"dA{l-1}"] = dA_prev
grads[f"dW{l}"], grads[f"db{l}"] = dW, db
return grads
def update_parameters_with_gd(self, grads, lr):

 L = self.get_num_layers() -1
for l in range(1, L+1):
self.set_params(f"W{l}", self.get_params(f"W{l}")- lr * grads[f"dW{l}"])
 self.set_params(f"b{l}", self.get_params(f"b{l}")- lr * grads[f"db{l}"])
 def initialize_adam(self):

 v = {}
s = {}
L = self.get_num_layers() -1
for l in range(1, L+1):
v[f"dW{l}"] = np.zeros(( self.get_params(f"W{l}" ).shape))
v[f"db{l}"] = np.zeros(( self.get_params(f"b{l}" ).shape))
s[f"dW{l}"] = np.zeros(( self.get_params(f"W{l}" ).shape))
s[f"db{l}"] = np.zeros(( self.get_params(f"b{l}" ).shape))
return v,s
def update_parameters_with_adam(self, grads, v, s, t, lr, beta1, beta2, epsilon):

 v_corr = {}
s_corr = {}
L = self.get_num_layers() -1
for l in range(1, L+1):
v[f"dW{l}"] = beta1*v[f"dW{l}"] +(1-beta1)*grads[f"dW{l}"]
v[f"db{l}"] = beta1*v[f"db{l}"] +(1-beta1)*grads[f"db{l}"]
v_corr[f"dW{l}"] = v[f"dW{l}"]/(1-beta1**t)
 v_corr[f"db{l}"] = v[f"db{l}"]/(1-beta1**t)
 s[f"dW{l}"] = beta2*s[f"dW{l}"] +(1-beta2)*grads[f"dW{l}"]**2
s[f"db{l}"] = beta2*s[f"db{l}"] +(1-beta2)*grads[f"db{l}"]**2
s_corr[f"dW{l}"] = s[f"dW{l}"]/(1-beta2**t)
 s_corr[f"db{l}"] = s[f"db{l}"]/(1-beta2**t)
 temp_W = v_corr[f"dW{l}"]/(s_corr[f"dW{l}"]**0.5 + epsilon)
 temp_b = v_corr[f"db{l}"]/(s_corr[f"db{l}"]**0.5 + epsilon)
 self.set_params(f"W{l}", self.get_params(f"W{l}")- lr*temp_W)
 self.set_params(f"b{l}", self.get_params(f"b{l}")- lr*temp_b)
 return v, s
def predict(self, X):

 m = X.shape[1]
p = np.zeros(( 1, m))
AL, _ = self.forward_propagation(X)
 for i in range(AL.shape[1]):
if AL[0,i] > 0.5:
p[0,i] = 1
else:
p[0,i] = 0
return np.int64(p)
 def fit(self, X, Y, batch_size, num_epochs, lr, min_lr=1e-5, lambd=0,
beta1=0.9, beta2=0.999, epsilon=1e-8, optimizer='adam', print_cost=False):

 costs = []
lr_0 = lr
t = 0
cost_prev = 1000
if optimizer == "gd":
pass
elif optimizer == "adam":
v, s = self.initialize_adam()
for epoch in range(num_epochs):
minibatches = random_mini_batches(X, Y, batch_size)
 for minibatch in minibatches: 
(minibatch_X, minibatch_Y)= minibatch
AL, caches = self.forward_propagation(minibatch_X)
 cost = self.compute_cost(AL, minibatch_Y, lambd)
 grads = self.backward_propagation(AL, minibatch_Y, caches, lambd)
 t = t + 1
if optimizer == "gd":
self.update_parameters_with_gd(grads, lr)
 elif optimizer == "adam":
v, s = self.update_parameters_with_adam(grads, v, s, t, lr,
beta1, beta2, epsilon)
 if cost > cost_prev and lr > min_lr:
lr = lr / 2
cost_prev = cost
if print_cost and epoch % 20 == 0:
print(f"Cost after epoch {epoch}: {cost}")
 if print_cost and epoch % 1 == 0:
costs.append(cost)
 if print_cost:
plt.plot(np.squeeze(costs))
plt.ylabel("cost")
 plt.xlabel("epochs(x 1)")
 plt.title(f"Learning rate = {lr_0} / {lr}")
 plt.show()
sub['ForecastId'] = sub['ForecastId'].astype(int)
 sub.columns = ['ForecastId','ConfirmedCases','Fatalities']<save_to_csv>
records_list = []
sub.to_csv("submission.csv",index=False )<load_from_csv>
c_model = Custom_model(layer_dims)
 lr = 1e-1
min_lr = 1e-7
optimizer = 'adam'
batch_size = m_train
num_epochs = 300
lambd = 5e+0
c_model.fit(X_train, Y_train, batch_size=batch_size, num_epochs=num_epochs,
lr=lr, lambd=lambd, optimizer=optimizer, print_cost=True)
 predict_train = c_model.predict(X_train)
 c_train_acc = round(np.sum(predict_train == Y_train)/m_train, 2)
 predict_val = c_model.predict(X_val)
 c_val_acc = round(np.sum(predict_val == Y_val)/m_val, 2)
 print(f"c_model: train accuracy: {c_train_acc * 100}%")
 print(f"c_model: val accuracy = {c_val_acc * 100}%")
 print(f'c_model: val error = {round(( 1 - c_val_acc)* m_val)} examples')
 record = {'layer_dims' : c_model.get_layer_dims() ,
'acc_train' : c_train_acc,
'acc_val' : c_val_acc,
'val_error' : round(( 1 - c_val_acc)* m_val),
'batch_size' : batch_size,
'num_epochs' : num_epochs,
'learning_rate': lr,
'min_lr' : min_lr,
'lambda' : lambd}
records_list.append(record)
 journal = pd.DataFrame(records_list)
 journal
<create_dataframe><EOS> 
c_model_final = Custom_model(layer_dims)
 X = np.concatenate(( X_train, X_val), axis=1)
 Y = np.concatenate(( Y_train, Y_val), axis=1)
 c_model_final.fit(X, Y, batch_size=(m_train + m_val), num_epochs=num_epochs,
lr=lr, min_lr=min_lr, lambd=lambd,
optimizer=optimizer, print_cost=True)
 prediction = c_model_final.predict(X_test)
 submission = pd.DataFrame({"PassengerId": test_data["PassengerId"],
"Survived": np.squeeze(prediction)} ).sort_values(by="PassengerId")
 submission.to_csv("submission.csv", index=False)
 submission.head()