Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<train_model>
print(os.listdir(".. /input"))
history3 = dual_lstm.fit([nlp_train, meta_train], labels, validation_split =.2,
epochs = 25, batch_size = 21, verbose = 1 )<predict_on_test>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )
submission_lstm2 = pd.DataFrame()
submission_lstm2['id'] = test_id
submission_lstm2['prob'] = dual_lstm.predict([nlp_test, meta_test])
 submission_lstm2['target'] = submission_lstm2['prob'].apply(lambda x: 0 if x <.5 else 1)
 submission_lstm2.head(10 )<define_variables>
tabla_completa = train_df.append(test_df, ignore_index=True )
BATCH_SIZE = 32
EPOCHS = 2
USE_META = True
ADD_DENSE = False
DENSE_DIM = 64
ADD_DROPOUT = False
DROPOUT =.2<install_modules>
tabla_completa["Cabin"] = tabla_completa["Cabin"].fillna("N")
 letras_cabinas = tabla_completa["Cabin"].str[0].unique().tolist()
!pip install --quiet transformers 
<categorify>
tabla_completa["Letra_Cabina"] = tabla_completa["Cabin"].str[0]
TOKENIZER = AutoTokenizer.from_pretrained("bert-large-uncased")
 enc = TOKENIZER.encode("Encode me!")
 dec = TOKENIZER.decode(enc)
 print("Encode: " + str(enc))
print("Decode: " + str(dec))<categorify>
dic_titulos_simples = {
"Mr" : "Mr",
"Mrs" : "Mrs",
"Miss" : "Miss",
"Master" : "Master",
"Don" : "Nobleza",
"Rev" : "Oficial",
"Dr" : "Oficial",
"Mme" : "Mrs",
"Ms" : "Mrs",
"Major" : "Oficial",
"Lady" : "Nobleza",
"Sir" : "Nobleza",
"Mlle" : "Miss",
"Col" : "Oficial",
"Capt" : "Oficial",
"Countess" : "Nobleza",
"Jonkheer" : "Nobleza",
"Dona" : "Nobleza"
}
tabla_completa["Titulo"] = tabla_completa["Name"].str.partition('.')[0].str.rpartition() [2].map(dic_titulos_simples )
def bert_encode(data,maximum_len):
input_ids = []
attention_masks = []
for i in range(len(data.text)) :
encoded = TOKENIZER.encode_plus(data.text[i],
add_special_tokens=True,
max_length=maximum_len,
pad_to_max_length=True,
return_attention_mask=True)
 input_ids.append(encoded['input_ids'])
 attention_masks.append(encoded['attention_mask'])
 return np.array(input_ids),np.array(attention_masks )<choose_model_class>
grupo = tabla_completa.groupby(['Sex','Pclass','Titulo'])
 tabla_completa['Age'] = grupo['Age'].apply(lambda x: x.fillna(x.median()))
def build_model(model_layer, learning_rate, use_meta = USE_META, add_dense = ADD_DENSE,
dense_dim = DENSE_DIM, add_dropout = ADD_DROPOUT, dropout = DROPOUT):
input_ids = tf.keras.Input(shape=(60,),dtype='int32')
 attention_masks = tf.keras.Input(shape=(60,),dtype='int32')
 meta_input = tf.keras.Input(shape =(meta_train.shape[1],))
transformer_layer = model_layer([input_ids,attention_masks])
 output = transformer_layer[1]
if use_meta:
output = tf.keras.layers.Concatenate()([output, meta_input])
 if add_dense:
print("Training with additional dense layer...")
 output = tf.keras.layers.Dense(dense_dim,activation='relu' )(output)
 if add_dropout:
print("Training with dropout...")
 output = tf.keras.layers.Dropout(dropout )(output)
 output = tf.keras.layers.Dense(1,activation='sigmoid' )(output)
 if use_meta:
print("Training with meta-data...")
 model = tf.keras.models.Model(inputs = [input_ids,attention_masks, meta_input],outputs = output)
 else:
print("Training without meta-data...")
 model = tf.keras.models.Model(inputs = [input_ids,attention_masks],outputs = output)
 model.compile(tf.keras.optimizers.Adam(lr=learning_rate), loss='binary_crossentropy', metrics=['accuracy'])
 return model<load_from_csv>
tabla_completa['Fare'] = tabla_completa['Fare'].fillna(tabla_completa['Fare'].median() )
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv' )<categorify>
tabla_completa['Num_Familiares'] = tabla_completa['SibSp'] + tabla_completa['Parch']
bert_large = TFAutoModel.from_pretrained('bert-large-uncased')
 TOKENIZER = AutoTokenizer.from_pretrained("bert-large-uncased")
 train_input_ids,train_attention_masks = bert_encode(train,60)
 test_input_ids,test_attention_masks = bert_encode(test,60)
 print('Train length:', len(train_input_ids))
print('Test length:', len(test_input_ids))
BERT_large = build_model(bert_large, learning_rate = 1e-5)
 BERT_large.summary()<train_model>
tabla_completa['Sex'] = tabla_completa['Sex'].map({"male": 0, "female":1})
 dummies_pclass = pd.get_dummies(tabla_completa['Pclass'], prefix="Pclass")
 dummies_titulo = pd.get_dummies(tabla_completa['Titulo'], prefix="Titulo")
 dummies_letra_cab = pd.get_dummies(tabla_completa['Letra_Cabina'], prefix="Letra_Cabina")
 dummies_grupo_edad = pd.get_dummies(tabla_completa['Grupo_Edad'], prefix="Grupo_Edad")
 dummies_tabla = pd.concat([tabla_completa, dummies_pclass, dummies_titulo, dummies_letra_cab,dummies_grupo_edad], axis=1)
 dummies_tabla.drop(['Pclass', 'Titulo', 'Cabin', 'Letra_Cabina', 'Grupo_Edad', 'Embarked', 'Name', 'Ticket'], axis=1, inplace=True )
history_bert = BERT_large.fit([train_input_ids,train_attention_masks, meta_train], train.target,
validation_split =.2, epochs = EPOCHS, callbacks = [checkpoint], batch_size = BATCH_SIZE )<predict_on_test>
train_df = dummies_tabla[ :len(train_df)]
test_df = dummies_tabla[(len(dummies_tabla)- len(test_df)) : ]
train_df['Survived'] = train_df['Survived'].astype(int)
 print("Train:
",train_df)
 print("Test:
",test_df )
BERT_large.load_weights('large_model.h5')
 preds_bert = BERT_large.predict([test_input_ids,test_attention_masks,meta_test] )<prepare_output>
test_df = test_df.reset_index(drop = True)
 train_df = train_df.reset_index(drop = True)
 train_dfX = train_df.drop(['PassengerId','Survived'], axis=1)
 train_dfY = train_df['Survived']
submission = pd.DataFrame(data=test_df['PassengerId'].copy())
 print(submission)
 test_df = test_df.drop(['PassengerId', 'Survived'], axis=1 )
submission_bert = pd.DataFrame()
submission_bert['id'] = test_id
submission_bert['prob'] = preds_bert
submission_bert['target'] = np.round(submission_bert['prob'] ).astype(int)
 submission_bert.head(10 )<save_to_csv>
sc = StandardScaler()
train_dfX = sc.fit_transform(train_dfX)
 test_df = sc.transform(test_df )
submission_bert = submission_bert[['id', 'target']]
submission_bert.to_csv('submission_bert.csv', index = False)
 print('Blended submission has been saved to disk' )<set_options>
train_dfX,val_dfX,train_dfY, val_dfY = train_test_split(train_dfX,train_dfY , test_size=0.1, stratify=train_dfY)
 print("Entrnamiento: ",train_dfX.shape)
 print("Validacion : ",val_dfX.shape )
plt.style.use('ggplot')
 warnings.filterwarnings('ignore')
<define_variables>
def func_model(arquitectura):
np.random.seed(42)
 random_seed = 42
first =True
inp = Input(shape=(train_dfX.shape[1],))
for capa in arquitectura:
if first:
x=Dense(capa, activation="relu", kernel_initializer=initializers.RandomNormal(seed=random_seed), bias_initializer='zeros' )(inp)
 first = False
else:
x=Dense(capa, activation="relu", kernel_initializer=initializers.RandomNormal(seed=random_seed), bias_initializer='zeros' )(x)
 x=Dense(1, activation="sigmoid", kernel_initializer=initializers.RandomNormal(seed=random_seed), bias_initializer='zeros' )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=optimizers.Adam(lr=0.0002), metrics=['binary_accuracy'])
 return model
def seed_everything(seed):
os.environ['PYTHONHASHSEED']=str(seed)
 tf.random.set_seed(seed)
 np.random.seed(seed)
 random.seed(seed)
 seed_everything(34 )<load_from_csv>
arq1 = [1024, 1024, 512]
model1 = None
model1 = func_model(arq1)
 train_history_tam1 = model1.fit(train_dfX, train_dfY, batch_size=32, epochs=epochs, validation_data=(val_dfX, val_dfY))
graf_model(train_history_tam1)
 precision(model1 )
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 train.head()<categorify>
arq2 = [1024, 512, 512]
model2 = None
model2 = func_model(arq2)
 train_history_tam2 = model2.fit(train_dfX, train_dfY, batch_size=32, epochs=epochs, validation_data=(val_dfX, val_dfY))
graf_model(train_history_tam2)
 precision(model2 )
test_id = test['id']
columns = {'id', 'location'}
train = train.drop(columns = columns)
 test = test.drop(columns = columns)
 train['keyword'] = train['keyword'].fillna('unknown')
 test['keyword'] = test['keyword'].fillna('unknown')
 train['text'] = train['text'] + ' ' + train['keyword']
test['text'] = test['text'] + ' ' + test['keyword']
columns = {'keyword'}
train = train.drop(columns = columns)
 test = test.drop(columns = columns)
 total = train.append(test )<feature_engineering>
arqFinal = [1024, 1024, 1024]
modelF = None
modelF = func_model(arqFinal)
 print(modelF.summary())
 train_history_tamF = modelF.fit(train_dfX, train_dfY, batch_size=32, epochs=epochs, validation_data=(val_dfX, val_dfY))
graf_model(train_history_tamF)
 precision(modelF, True )
total['unique word count'] = total['text'].apply(lambda x: len(set(x.split())))
 total['stopword count'] = total['text'].apply(lambda x: len([i for i in x.lower().split() if i in wordcloud.STOPWORDS]))
total['stopword ratio'] = total['stopword count'] / total['word count']
total['punctuation count'] = total['text'].apply(lambda x: len([i for i in str(x)if i in string.punctuation]))
train = total[:len(train)]
disaster = train['target'] == 1
fig, axes = plt.subplots(4, figsize=(20, 30))
graph1 = sns.kdeplot(train.loc[~disaster]['unique word count'], shade = True, label = 'Not Disaster', ax=axes[0])
 graph1 = sns.kdeplot(train.loc[disaster]['unique word count'], shade = True, label = 'Disaster', ax=axes[0])
 graph1.set_title('Distribution of Unique Word Count')
 graph2 = sns.kdeplot(train.loc[~disaster]['stopword count'], shade = True, label = 'Not Disaster', ax=axes[1])
 graph2 = sns.kdeplot(train.loc[disaster]['stopword count'], shade = True, label = 'Disaster', ax=axes[1])
 graph2.set_title('Distribution of Stopword Word Count')
 graph3 = sns.kdeplot(train.loc[~disaster]['punctuation count'], shade = True, label = 'Not Disaster', ax=axes[2], bw = 1)
 graph3 = sns.kdeplot(train.loc[disaster]['punctuation count'], shade = True, label = 'Disaster', ax=axes[2], bw = 1)
 graph3.set_title('Distribution of Punctuation Count')
 graph4 = sns.kdeplot(train.loc[~disaster]['stopword ratio'], shade = True, label = 'Not Disaster', ax=axes[3], bw =.05)
 graph4 = sns.kdeplot(train.loc[disaster]['stopword ratio'], shade = True, label = 'Disaster', ax=axes[3], bw =.05)
 graph4.set_title('Distribution of Stopword Ratio')
 fig.tight_layout()
plt.show()<string_transform>
def func_model_reg() :
np.random.seed(42)
 random_seed = 42
inp = Input(shape=(train_dfX.shape[1],))
x=Dropout(0.1 )(inp)
 x=Dense(1024, activation="relu", kernel_initializer=initializers.RandomNormal(seed=random_seed), bias_initializer='zeros', kernel_regularizer=regularizers.l2(0.01))(x)
 x=Dropout(0.7 )(x)
 x=Dense(1024, activation="relu", kernel_initializer=initializers.RandomNormal(seed=random_seed), bias_initializer='zeros', kernel_regularizer=regularizers.l2(0.01))(x)
 x=Dropout(0.7 )(x)
 x=Dense(512, activation="relu", kernel_initializer=initializers.RandomNormal(seed=random_seed), bias_initializer='zeros', kernel_regularizer=regularizers.l2(0.01))(x)
 x=Dropout(0.9 )(x)
 x=Dense(1, activation="sigmoid", kernel_initializer=initializers.RandomNormal(seed=random_seed), bias_initializer='zeros' )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=optimizers.Adam(lr=0.0002), metrics=['binary_accuracy'])
 return model
def remove_punctuation(x):
return x.translate(str.maketrans('', '', string.punctuation))
def remove_stopwords(x):
return ' '.join([i for i in x.split() if i not in wordcloud.STOPWORDS])
 def remove_less_than(x):
return ' '.join([i for i in x.split() if len(i)> 3])
 def remove_non_alphabet(x):
return ' '.join([i for i in x.split() if i.isalpha() ])
 def strip_all_entities(x):
return ' '.join(re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)"," ",x ).split() )<drop_column>
modelReg = None
modelReg = func_model_reg()
train_history_tamReg = modelReg.fit(train_dfX, train_dfY, batch_size=32, epochs=epochs, validation_data=(val_dfX, val_dfY))
graf_model(train_history_tamReg)
 precision(modelReg )
<string_transform><EOS> 
y_test = modelReg.predict(test_df)
 submission['Survived'] = y_test.round().astype(int)
 submission.to_csv('submission.csv', index=False )