Kaggle competition: Titanic - Machine Learning from Disaster
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: x.lower())
 test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: x.lower() )<feature_engineering>
train = pd.read_csv("/kaggle/input/titanic/train.csv")
 test = pd.read_csv("/kaggle/input/titanic/test.csv" )
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: clean_text(x))<feature_engineering>
train = train.drop(columns=['Name','Cabin','Ticket'])
 test = test.drop(columns=['Name','Cabin','Ticket'] )
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_numbers(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: clean_numbers(x))<feature_engineering>
train['Embarked_S'] =(train['Embarked'] == 'S' ).astype(int)
 train['Embarked_C'] =(train['Embarked'] == 'C' ).astype(int)
 train['Embarked_Q'] =(train['Embarked'] == 'Q' ).astype(int)
 train['Gender'] =(train['Sex'] == 'male' ).astype(int )
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))<prepare_x_and_y>
test['Embarked_S'] =(test['Embarked'] == 'S' ).astype(int)
 test['Embarked_C'] =(test['Embarked'] == 'C' ).astype(int)
 test['Embarked_Q'] =(test['Embarked'] == 'Q' ).astype(int)
 test['Gender'] =(test['Sex'] == 'male' ).astype(int )
train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_<string_transform>
train = train.drop(columns = ['Sex'])
 test = test.drop(columns = ['Sex'])
 train = train.drop(columns = ['Embarked'])
 test = test.drop(columns = ['Embarked'] )
def tokenize_and_split(train_X,test_X):
tokenizer = Tokenizer(num_words=max_features,oov_token = 'xxunk',filters='')
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
trn_idx = np.random.permutation(len(train_X))
train_X = train_X[trn_idx]
train_y = train_y[trn_idx]
features, test_features = make_stat_features()
train_features = features[trn_idx]
return train_X, test_X, train_y, train_features, test_features ,tokenizer.word_index<normalization>
train.fillna(0, inplace=True)
 test.fillna(0, inplace=True )
def make_stat_features() :
train_features = train[['num_unique_words','words_vs_unique','total_length','capitals',
'caps_vs_length','num_words']].fillna(0)
 test_features = test[['num_unique_words','words_vs_unique','total_length','capitals',
'caps_vs_length','num_words']].fillna(0)
 ss = StandardScaler()
ss.fit(np.vstack(( train_features, test_features)))
 train_features = ss.transform(train_features)
 test_features = ss.transform(test_features)
 return train_features, test_features<split>
X = train.drop(columns=['Survived'])
 y = train['Survived']
x_train, x_test, y_train, train_features, test_features , word_index = tokenize_and_split(train_X,test_X )<compute_train_metric>
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42 )
def load_embedding(path, word_index,emb_mean, emb_std):
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(path, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<load_pretrained>
model_1 = LGBMClassifier(learning_rate=0.01, n_estimators=1000, max_depth=None)
 model_2 = XGBClassifier(learning_rate=0.01, n_estimators=1000, max_depth=None)
 model_3 = AdaBoostClassifier(learning_rate=0.01, n_estimators=1000)
 model_4 = RandomForestClassifier(n_estimators=1000, random_state=42 )
if debug:
paragram_embeddings = np.random.randn(120000,300)
 glove_embeddings = np.random.randn(120000,300)
 else:
glove_embeddings = load_embedding('.. /input/quora-insincere-questions-classification/embeddings/glove.840B.300d/glove.840B.300d.txt',word_index,-0.005838499,0.48782197)
 paragram_embeddings = load_embedding('.. /input/quora-insincere-questions-classification/embeddings/paragram_300_sl999/paragram_300_sl999.txt',word_index,-0.0053247833,0.49346462)
 embedding_matrix = np.concatenate(( glove_embeddings, paragram_embeddings), axis=1 )<init_hyperparams>
estimators = []
estimators.append(( 'lgbm',model_1))
estimators.append(( 'xgb',model_2))
estimators.append(( 'adaboost',model_3))
estimators.append(( 'clf',model_4))
class AttentionBlock(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(AttentionBlock, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )<choose_model_class>
hybrid_model = StackingClassifier(estimators )
class EmbeddingDropout(nn.Module):
def __init__(self, embedding_matrix, max_features = 120000, embedding_size = 300):
super(EmbeddingDropout,self ).__init__()
self.embedding = nn.Embedding(max_features, embedding_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_drop = nn.Dropout2d(0.1)
 def forward(self,x):
h_embeddings = self.embedding(x)
 h_embeddings = h_embeddings.unsqueeze(0)
 h_embeddings = h_embeddings.permute(1,3,0,2)
 h_embeddings = self.embedding_drop(h_embeddings)
 h_embeddings = h_embeddings.permute(2,0,3,1)
 h_embeddings = h_embeddings.squeeze(0)
 return h_embeddings<define_search_model>
hybrid_model.fit(X_train, y_train )
class Body(nn.Module):
def __init__(self, embedding_size= 300, hidden_size= 128):
super(Body,self ).__init__()
self.lstm = nn.LSTM(embedding_size*num_embeddings, hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)
 self.hidden= hidden_size
for name, param in self.lstm.named_parameters() :
if 'bias' in name:
nn.init.constant_(param, 0.0)
 elif 'weight_ih' in name:
nn.init.kaiming_normal_(param)
 elif 'weight_hh' in name:
nn.init.orthogonal_(param)
 for name, param in self.gru.named_parameters() :
if 'bias' in name:
nn.init.constant_(param, 0.0)
 elif 'weight_ih' in name:
nn.init.kaiming_normal_(param)
 elif 'weight_hh' in name:
nn.init.orthogonal_(param)
 self.lstm_attention = AttentionBlock(hidden_size*2, maxlen)
 self.gru_attention = AttentionBlock(hidden_size*2, maxlen)
 def forward(self,x):
h_lstm, _ = self.lstm(x)
 h_gru, _ = self.gru(h_lstm)
 h_lstm_att = self.lstm_attention(h_lstm)
 h_gru_att = self.gru_attention(h_gru)
 return h_lstm_att, h_gru_att, h_lstm, h_gru<feature_engineering>
pred = hybrid_model.predict(X_test )
class Extractor(nn.Module):
def __init__(self,maxlen= 70,hidden_size= 128,out= 64,len_feats= 6):
super(Extractor,self ).__init__()
self.conv = nn.Conv1d(maxlen,out,kernel_size= 1,stride= 2)
 self.stat = nn.Linear(len_feats,hidden_size)
 def forward(self,h_lstm,h_gru,stat_features):
conv_out = self.conv(h_lstm)
 l_maxpool, _ = torch.max(conv_out,1)
 g_avgpool = torch.mean(h_gru,1)
 statistical_features = self.stat(stat_features)
 return l_maxpool, g_avgpool, statistical_features<choose_model_class>
accuracy_score(pred, y_test )
class Head(nn.Module):
def __init__(self,embedding_size= 300,intermediate_layer=64,maxlen=70,hidden_size=128):
super(Head,self ).__init__()
self.linear = nn.Linear(hidden_size*8,intermediate_layer)
 self.dropout = nn.Dropout(0.15)
 self.bn = nn.BatchNorm1d(intermediate_layer)
 self.output = nn.Linear(intermediate_layer, 1)
 def forward(self,x):
return self.output(self.bn(self.dropout(self.linear(x))))<choose_model_class>
test['Survived'] = actual_pred = hybrid_model.predict(test )
<choose_model_class><EOS> 
test[['PassengerId','Survived']].to_csv('submission.csv', index=False )