Kaggle competition: Titanic - Machine Learning from Disaster
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
train_data = pd.read_csv('/kaggle/input/titanic/train.csv')
 test_data = pd.read_csv('/kaggle/input/titanic/test.csv' )
def build_model(lr=0.0, lr_d=0.0, units=64, spatial_dr=0.0,
dense_units=0, dr=0.1, conv_size=32, epochs=20):
file_path = "best_model.hdf5"
check_point = ModelCheckpoint(file_path, monitor="val_loss", verbose=1, save_best_only=True, mode="min")
 early_stop = EarlyStopping(monitor="val_loss", mode="min", patience=3)
 sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
 embedding_layer = Embedding(*embedding_matrix.shape,
weights=[embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=True)
 x = embedding_layer(sequence_input)
 x = SpatialDropout1D(spatial_dr )(x)
 x = Bidirectional(CuDNNGRU(units, return_sequences=True))(x)
 x = Conv1D(conv_size, 2, padding="valid", kernel_initializer="he_uniform" )(x)
 avg_pool1 = GlobalAveragePooling1D()(x)
 max_pool1 = GlobalMaxPooling1D()(x)
 x = concatenate([avg_pool1, max_pool1])
 x = BatchNormalization()(x)
 x = Dense(int(dense_units / 2), activation='relu' )(x)
 x = Dropout(dr )(x)
 preds = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=sequence_input, outputs=preds)
 model.compile(loss="binary_crossentropy", optimizer=Adam(lr=lr, decay=lr_d),
metrics=['acc', true_positives, possible_positives, predicted_positives])
 model.summary()
history = model.fit(train_text, Y_train, batch_size=1024, epochs=epochs, validation_split=0.1,
verbose=1, callbacks=[check_point, early_stop, F1Callback() ])
 return model<predict_on_test>
full_data['Fare']=full_data['Fare'].map(lambda x: np.log(x)if x>0 else 0 )
model = build_model(lr=1e-3, lr_d=1e-7, units=64, spatial_dr=0.2, dense_units=64, dr=0.1, conv_size=64, epochs=10)
 pred = model.predict(test_text, batch_size=1024 )<compute_train_metric>
full_data['Cabin'] = full_data['Cabin'].fillna('Unkown')
 full_data['Cabin'].head()
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result 
<save_to_csv>
full_data[full_data['Embarked'].isnull() ]
pred =(pred[:,0] > 0.5 ).astype(np.int)
 submission = pd.read_csv('.. /input/sample_submission.csv', index_col='qid')
 submission['prediction'] = pred
submission.reset_index(drop=False, inplace=True)
 submission.to_csv('submission.csv', index=False)
 submission.head()<count_values>
full_data['Embarked'].value_counts()
pd.Series(pred ).value_counts().to_frame().T / len(pred )<import_modules>
full_data['Embarked']=full_data['Embarked'].fillna('S' )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import preprocessing
from xgboost import XGBRegressor
from sklearn.tree import DecisionTreeClassifier<import_modules>
full_data[full_data['Fare'].isnull() ]
from fastai import *
from fastai.tabular import *
from fastai.callbacks import *<define_variables>
full_data['Fare'] = full_data['Fare'].fillna(full_data[(full_data['Pclass']==3)&(full_data['Embarked']=='S')&(full_data['Cabin']=='U')]['Fare'].mean() )
path1 = "/kaggle/input/covid19-global-forecasting-week-4"
path2 = "/kaggle/input/covid19-demographic-predictors"
path3 = "/kaggle/input/covid19-country-data-wk3-release"
path4 = "/kaggle/input/countryinfo"<load_from_csv>
full_data['Title']=full_data['Name'].map(lambda x:x.split(',')[1].split('.')[0].strip())
 full_data['Title'].value_counts()
train_df = pd.read_csv(f"{path1}/train.csv", parse_dates=['Date'])
 test_df = pd.read_csv(f"{path1}/test.csv", parse_dates=['Date'] )<feature_engineering>
TitleDict={}
TitleDict['Mr']='Mr'
TitleDict['Mlle']='Miss'
TitleDict['Miss']='Miss'
TitleDict['Master']='Master'
TitleDict['Jonkheer']='Master'
TitleDict['Mme']='Mrs'
TitleDict['Ms']='Mrs'
TitleDict['Mrs']='Mrs'
TitleDict['Don']='Royalty'
TitleDict['Sir']='Royalty'
TitleDict['the Countess']='Royalty'
TitleDict['Dona']='Royalty'
TitleDict['Lady']='Royalty'
TitleDict['Capt']='Officer'
TitleDict['Col']='Officer'
TitleDict['Major']='Officer'
TitleDict['Dr']='Officer'
TitleDict['Rev']='Officer'
full_data['Title']=full_data['Title'].map(TitleDict)
 full_data['Title'].value_counts()
add_datepart(train_df, 'Date', drop=False )<feature_engineering>
TickCountDict = {}
TickCountDict = full_data['Ticket'].value_counts()
TickCountDict.head()
add_datepart(test_df, 'Date', drop=False )<drop_column>
full_data['TickGroup'] = full_data['Ticket'].map(TickCountDict)
 full_data['TickGroup'].head()
missed = "NA"
def State(state, country):
if state == missed: return country
return state<load_from_csv>
AgePre = full_data[['Age','Parch','Pclass','SibSp','Title','familySize','TickGroup']]
AgePre = pd.get_dummies(AgePre)
 ParAge = pd.get_dummies(AgePre['Parch'],prefix='Parch')
 SibAge = pd.get_dummies(AgePre['SibSp'],prefix='SibSp')
 PclAge = pd.get_dummies(AgePre['Pclass'],prefix='Pclass')
 AgeCorrDf = pd.DataFrame()
AgeCorrDf = AgePre.corr()
AgeCorrDf['Age'].sort_values()
metadata_df = pd.read_csv(f"{path3}/Data Join - RELEASE.csv", thousands=',' )<load_from_csv>
AgePre = pd.concat([AgePre,ParAge,SibAge,PclAge],axis=1)
 AgePre.head()
country_df = pd.read_csv(f"{path4}/covid19countryinfo.csv",thousands=",", parse_dates=['quarantine', 'schools', 'publicplace', 'gathering', 'nonessential'] )<load_from_csv>
Age_train = AgePre[AgePre['Age'].notnull() ]
Age_test = AgePre[AgePre['Age'].isnull() ]
AgeKnown_X = Age_train.drop(['Age'],axis=1)
 AgeKnown_Y = Age_train['Age']
testinfo = pd.read_csv(f'{path4}/covid19tests.csv', thousands="," )<drop_column>
Age_X_test = Age_test.drop(['Age'],axis=1 )
country_df.rename(columns={'region': 'Province_State', 'country': 'Country_Region'}, inplace=True)
 testinfo.rename(columns={'region': 'Province_State', 'country': 'Country_Region'}, inplace=True)
 testinfo = testinfo.drop(['alpha3code', 'alpha2code', 'date'], axis=1 )<define_variables>
rfr=RandomForestRegressor(random_state=None,n_estimators=500,n_jobs=-1)
 rfr.fit(AgeKnown_X,AgeKnown_Y)
 print(rfr.score(AgeKnown_X,AgeKnown_Y))
group_cols = ['Country_Region', 'Province_State']<groupby>
AgeUnKnown_Y = rfr.predict(Age_X_test)
 full_data.loc[full_data['Age'].isnull() ,['Age']] = AgeUnKnown_Y
full_data.info()
group = train_df[train_df["Fatalities"] > 1].groupby(group_cols)
 res_df =(group.Fatalities.last() / group.ConfirmedCases.last() ).reset_index()
fatalities = res_df.rename(columns={0 : 'FatalityRate'} )<groupby>
fullSel = fullSel.drop(['Family','SibSp','TickGroup','Parch'],axis=1)
 fullSel = pd.get_dummies(fullSel)
 PclassDf = pd.get_dummies(full_data['Pclass'],prefix='Pclass')
 TickGroupDf = pd.get_dummies(full_data['TickGroup'],prefix='TickGroup')
 familySizeDf = pd.get_dummies(full_data['familySize'],prefix='familySize')
 fullSel=pd.concat([fullSel,PclassDf,TickGroupDf,familySizeDf],axis=1 )
group = train_df[train_df['ConfirmedCases'] >= 1].groupby(group_cols)
 first_confirmed = group.Dayofyear.first().reset_index().rename(columns={'Dayofyear': "First_Confirmed"} )<groupby>
kfold = StratifiedKFold(n_splits=10)
 experData = fullSel[fullSel['Survived'].notnull() ]
preData = fullSel[fullSel['Survived'].isnull() ]
experData_X = experData.drop('Survived',axis=1)
 experData_y = experData['Survived']
preData_X = preData.drop('Survived',axis=1)
 modelLR = LogisticRegression()
LR_param_grid = {'C' : [1,2,3],
'penalty':['l1','l2']}
modelgsLR = GridSearchCV(modelLR,param_grid = LR_param_grid, cv=kfold,
scoring="accuracy", n_jobs= -1, verbose = 1)
 modelgsLR.fit(experData_X,experData_y )
group = train_df[train_df['ConfirmedCases'] >= 50].groupby(group_cols)
 first_50= group.Dayofyear.first().reset_index().rename(columns={'Dayofyear': "First_50"} )<groupby>
print("Accuracy: ", modelgsLR.best_score_ )
group = train_df[train_df['ConfirmedCases'] >= 100].groupby(group_cols)
 first_hundred = group.Dayofyear.first().reset_index().rename(columns={'Dayofyear': "First_Hundred"} )<merge>
GBC = GradientBoostingClassifier()
gb_param_grid = {'loss' : ["deviance"],
'n_estimators' : [100,200,300],
'learning_rate': [0.1, 0.05, 0.01],
'max_depth': [4, 8],
'min_samples_leaf': [100,150],
'max_features': [0.3, 0.1]
}
modelgsGBC = GridSearchCV(GBC,param_grid = gb_param_grid, cv=kfold,
scoring="accuracy", n_jobs= -1, verbose = 1)
 modelgsGBC.fit(experData_X,experData_y )
<merge><EOS> 
GBCpreData_y=modelgsGBC.predict(preData_X)
 GBCpreData_y=GBCpreData_y.astype(int)
 GBCpreResultDf=pd.DataFrame()
GBCpreResultDf['PassengerId']=full_data['PassengerId'][full_data['Survived'].isnull() ]
GBCpreResultDf['Survived']=GBCpreData_y
GBCpreResultDf
GBCpreResultDf.to_csv('submission.csv',index=False )