Kaggle competition: Titanic - Machine Learning from Disaster
paras = []
labels = []
texts = []<string_transform>
train=pd.read_csv('/kaggle/input/titanic/train.csv')
 test=pd.read_csv('/kaggle/input/titanic/test.csv')
 train.head()
sent_lens = []
sent_nums = []
for idx in tqdm(range(train_text.shape[0])) :
text = train_text.text[idx]
texts.append(text)
 sentences = tokenize.sent_tokenize(text)
 sent_nums.append(len(sentences))
for sent in sentences:
sent_lens.append(len(text_to_word_sequence(sent)))
 paras.append(sentences )<define_variables>
df = train.copy()
val_paras = []
val_labels = []<define_variables>
testo =test.copy()
test_paras = []
test_labels = []<string_transform>
df.drop(columns=['Name', 'Ticket', 'Cabin'], axis=1, inplace=True)
 test.drop(columns=['Name', 'Ticket', 'Cabin'], axis=1, inplace=True )
for idx in range(test.shape[0]):
text = test[idx]
sentences = tokenize.sent_tokenize(text)
 test_paras.append(sentences )<feature_engineering>
df.isnull().sum()
tokenizer = Tokenizer(num_words=max_features, oov_token=True)
 tokenizer.fit_on_texts(texts )<feature_engineering>
df['Age'].fillna(df['Age'].median() , inplace=True)
 df['Embarked'].fillna(df['Embarked'].mode() [0], inplace=True)
x_train = np.zeros(( len(texts), max_senten_num, max_senten_len), dtype='int32')
 for i, sentences in tqdm(enumerate(paras)) :
tokenized_sent = tokenizer.texts_to_sequences(sentences)
 padded_seq = pad_sequences(tokenized_sent, maxlen=max_senten_len, padding='post', truncating='post')
 for j, seq in enumerate(padded_seq):
if(j < max_senten_num):
x_train[i,j,:] = seq
else:
break<categorify>
df.isnull().sum()
<categorify>
df[df.Fare.isnull() ]
test_data = np.zeros(( test.shape[0], max_senten_num, max_senten_len), dtype='int32')
 for i, sentences in enumerate(test_paras):
tokenized_sent = tokenizer.texts_to_sequences(sentences)
 padded_seq = pad_sequences(tokenized_sent, maxlen=max_senten_len, padding='post', truncating='post')
 for j, seq in enumerate(padded_seq):
if(j < max_senten_num):
test_data[i,j,:] = seq
else:
break<count_unique_values>
df.isnull().sum()
word_index = tokenizer.word_index
print('Total %s unique tokens.' % len(word_index))<import_modules>
test.isnull().sum()
import os<statistical_test>
test.isnull().sum()
gc.collect()
word_index = tokenizer.word_index
max_features = len(word_index)+1
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word.lower() , np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100 and o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word.lower() , np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100 and o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<normalization>
print('Duplicated data =',df.duplicated().sum() )
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_3 = load_para(word_index)
 embedding_matrix = np.mean(( embedding_matrix_1, embedding_matrix_3), axis=0)
 del embedding_matrix_1, embedding_matrix_3
gc.collect()
np.shape(embedding_matrix )<train_model>
df[['Pclass','Survived']].groupby(['Pclass'],as_index = False ).mean().sort_values(by = 'Survived', ascending = False )
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
 def f1(y_true, y_pred):

 def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))<categorify>
df[['Sex','Survived']].groupby(['Sex'],as_index = False ).mean()
def han_model(embedding_matrix):
nb_words = embedding_matrix.shape[0]
embedding_layer = Embedding(nb_words, embed_size, weights=[embedding_matrix])
 word_input = Input(shape=(max_senten_len,), dtype='float32')
 word_sequences = embedding_layer(word_input)
 word_lstm = Bidirectional(CuDNNLSTM(64, return_sequences=True))(word_sequences)
 word_att = AttentionWithContext()(word_lstm)
 word_att = ReLU()(word_att)
 wordEncoder = Model(word_input, word_att)
 sent_input = Input(shape=(max_senten_num, max_senten_len), dtype='float32')
 sent_encoder = TimeDistributed(wordEncoder )(sent_input)
 sent_enc_avg = Lambda(lambda x: K.sum(x, axis=1))(sent_encoder)
 sent_enc_avg = ReLU()(sent_enc_avg)
 pred1 = Dense(1, activation='sigmoid' )(sent_enc_avg)
 sent_lstm = Bidirectional(CuDNNLSTM(128, return_sequences=True))(sent_encoder)
 sent_att = AttentionWithContext()(sent_lstm)
 sent_att = ReLU()(sent_att)
 pred2 = Dense(1, activation='sigmoid' )(sent_att)
 preds = Average()([pred1, pred2])
 model = Model(sent_input, preds)
 model.compile(loss='binary_crossentropy',optimizer=Adam() ,metrics=[f1])
 return model<predict_on_test>
df[['SibSp','Survived']].groupby(['SibSp'] ).mean()
def train_pred(model, train_X, train_y, val_X, val_y, epochs=2, callback=None, batch_size=512):
print(train_X.dtype, train_y.dtype)
 h = model.fit(train_X, train_y, batch_size=batch_size, epochs=epochs, validation_data=(val_X, val_y), callbacks = callback, verbose=1)
 model.load_weights(filepath)
 pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)
 pred_test_y = model.predict([test_data], batch_size=1024, verbose=0)
 print('=' * 60)
 return pred_val_y, pred_test_y<import_modules>
df[['Parch','Survived']].groupby(['Parch'] ).mean()
from sklearn.model_selection import GridSearchCV, StratifiedKFold<compute_test_metric>
LabelEncoder = preprocessing.LabelEncoder()
df['Embarked'] = LabelEncoder.fit_transform(df['Embarked'])
 test['Embarked'] = LabelEncoder.transform(test['Embarked'])
 df['Sex'] = LabelEncoder.fit_transform(df['Sex'])
 test['Sex'] = LabelEncoder.fit_transform(test['Sex'])
 y = df['Survived']
X = df.drop(['Survived'],axis=1 )
search_result = threshold_search(y_train, train_meta )<save_to_csv>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25 )
pred_test_y =(test_meta>search_result['threshold'] ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<set_options>
scaler = MinMaxScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test)
 test = scaler.fit_transform(test )
gc.collect()<import_modules>
def perform_model(model, X_train, y_train, X_test, y_test, class_labels, cm_normalize=True, \
print_cm=True, cm_cmap=plt.cm.Greens):
results = dict()
train_start_time = datetime.now()
print('training the model.. ')
 model.fit(X_train, y_train)
 print('Done
')
 train_end_time = datetime.now()
results['training_time'] = train_end_time - train_start_time
print('training_time(HH:MM:SS.ms)- {}
'.format(results['training_time']))
print('Predicting test data')
 test_start_time = datetime.now()
y_pred = model.predict(X_test)
 test_end_time = datetime.now()
print('Done
')
 results['testing_time'] = test_end_time - test_start_time
print('testing time(HH:MM:SS:ms)- {}
'.format(results['testing_time']))
results['predicted'] = y_pred
accuracy = accuracy_score(y_true=y_test, y_pred=y_pred)
 results['accuracy'] = accuracy
print('---------------------')
 print('| Accuracy |')
 print('---------------------')
 print('
{}
'.format(accuracy))
cm = confusion_matrix(y_test, y_pred)
 results['confusion_matrix'] = cm
if print_cm:
print('--------------------')
 print('| Confusion Matrix |')
 print('--------------------')
 print('
{}'.format(cm))
plt.figure(figsize=(4,4))
plt.grid(b=False)
 plot_confusion_matrix(cm, classes=class_labels, normalize=True, title='Normalized confusion matrix', cmap = cm_cmap)
 plt.show()
print('-------------------------')
 print('| Classifiction Report |')
 print('-------------------------')
 classification_Report = classification_report(y_test, y_pred)
 results['classification_Report'] = classification_Report
print(classification_Report)
 results['model'] = model
return results
import os
import random
import re
import time
from collections import Counter
from itertools import chain
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
from sklearn.metrics import f1_score, roc_auc_score
from sklearn.model_selection import StratifiedKFold, KFold
from sklearn.utils import shuffle
from torch import optim
from torch.utils.data import Dataset, Sampler, DataLoader
from tqdm import tqdm<define_variables>
def print_grid_search_attributes(model):
print('--------------------------')
 print('| Best Estimator |')
 print('--------------------------')
 print('
\t{}
'.format(model.best_estimator_))
print('--------------------------')
 print('| Best parameters |')
 print('--------------------------')
 print('\tParameters of best estimator :
\t{}
'.format(model.best_params_))
print('---------------------------------')
 print('| No of CrossValidation sets |')
 print('--------------------------------')
 print('
\tTotal numbre of cross validation sets: {}
'.format(model.n_splits_))
print('--------------------------')
 print('| Best Score |')
 print('--------------------------')
 print('
\tAverage Cross Validate scores of best estimator :
\t{}
'.format(model.best_score_))
embedding_glove = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
embedding_fasttext = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
embedding_para = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
embedding_w2v = '.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'
train_path = '.. /input/train.csv'
test_path = '.. /input/test.csv'
mispell_dict = {"ain't": "is not", "aren't": "are not", "can't": "cannot", "'cause": "because",
"could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not",
"don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not",
"he'd": "he would", "he'll": "he will", "he's": "he is", "how'd": "how did",
"how'd'y": "how do you", "how'll": "how will", "how's": "how is", "i'd": "i would",
"i'd've": "i would have", "i'll": "i will", "i'll've": "I will have", "i'm": "i am",
"i've": "I have", "isn't": "is not", "it'd": "it would",
"it'd've": "it would have", "it'll": "it will", "it'll've": "it will have", "it's": "it is",
"let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have",
"mightn't": "might not", "mightn't've": "might not have", "must've": "must have",
"mustn't": "must not", "mustn't've": "must not have", "needn't": "need not",
"needn't've": "need not have", "o'clock": "of the clock", "oughtn't": "ought not",
"oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not",
"shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have",
"she'll": "she will", "she'll've": "she will have", "she's": "she is",
"should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have",
"so've": "so have", "so's": "so as", "this's": "this is", "that'd": "that would",
"that'd've": "that would have", "that's": "that is", "there'd": "there would",
"there'd've": "there would have", "there's": "there is", "here's": "here is",
"they'd": "they would", "they'd've": "they would have", "they'll": "they will",
"they'll've": "they will have", "they're": "they are", "they've": "they have",
"to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have",
"we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have",
"weren't": "were not", "what'll": "what will", "what'll've": "what will have",
"what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is",
"when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have",
"who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have",
"why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not",
"won't've": "will not have", "would've": "would have", "wouldn't": "would not",
"wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would",
"y'all'd've": "you all would have", "y'all're": "you all are", "y'all've": "you all have",
"you'd": "you would", "you'd've": "you would have", "you'll": "you will",
"you'll've": "you will have", "you're": "you are", "you've": "you have", 'colour': 'color',
'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling',
'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor',
'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize',
'youtu ': 'youtube ', 'qoura': 'quora', 'sallary': 'salary', 'whta': 'what',
'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can',
'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doi': 'do I',
'thebest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation',
'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis',
'etherium': 'ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017',
'2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess',
"whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization',
'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
puncts = ''!"
'
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2",
"—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '”': '"', '“': '"', "£": "e",
'∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta',
'∅': '', '³': '3', 'π': 'pi', '\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for p in puncts:
punct_mapping[p] = ' %s ' % p
p = re.compile('(\[ math \] ).+(\[ / math \])')
 p_space = re.compile(r'[^\x20-\x7e]' )<set_options>
labels = ['0','1']
def set_seed(seed):
np.random.seed(seed)
 torch.manual_seed(seed + 1)
 if torch.cuda.is_available() : torch.cuda.manual_seed_all(seed + 2)
 random.seed(seed + 4)
<string_transform>
parameters = {"C":np.logspace(-3,3,7), "penalty":["l1","l2"]}
logreg=LogisticRegression()
lr_grid = GridSearchCV(logreg,param_grid=parameters, n_jobs=-1)
 lr_grid_results = perform_model(lr_grid, X_train, y_train, X_test, y_test, class_labels=labels)
 print_grid_search_attributes(lr_grid_results['model'] )
def clean_text(text):
text = p.sub(' [ math ] ', text)
 text = p_space.sub(r'', text)
 for punct in punct_mapping:
if punct in text:
text = text.replace(punct, punct_mapping[punct])
 tokens = []
for token in text.split() :
token = mispell_dict.get(token.lower() , token)
 tokens.append(token)
 text = ' '.join(tokens)
 return text
def load_data(train_path=train_path, test_path=test_path, debug=False):
train_df = pd.read_csv(train_path)
 test_df = pd.read_csv(test_path)
 if debug:
train_df = train_df[:10000]
test_df = test_df[:10000]
s = time.time()
train_df['question_text'] = train_df['question_text'].apply(clean_text)
 test_df['question_text'] = test_df['question_text'].apply(clean_text)
 print('preprocssing {}s'.format(time.time() - s))
return train_df, test_df<count_duplicates>
parameters = {'max_depth':np.arange(3,10,2)}
dt = DecisionTreeClassifier()
dt_grid = GridSearchCV(dt,param_grid=parameters, n_jobs=-1)
 dt_grid_results = perform_model(dt_grid, X_train, y_train, X_test, y_test, class_labels=labels)
 print_grid_search_attributes(dt_grid_results['model'] )
def build_counter(sents, splited=False):
counter = Counter()
for sent in tqdm(sents, ascii=True, desc='building conuter'):
if splited:
counter.update(sent)
 else:
counter.update(sent.split())
 return counter
def build_vocab(counter, max_vocab_size):
vocab = {'token2id': {'<PAD>': 0, '<UNK>': max_vocab_size + 1}}
vocab['token2id'].update(
 {token: _id + 1 for _id,(token, count)in
tqdm(enumerate(counter.most_common(max_vocab_size)) , desc='building vocab')})
 vocab['id2token'] = {v: k for k, v in vocab['token2id'].items() }
return vocab
def tokens2ids(tokens, token2id):
seq = []
for token in tokens:
token_id = token2id.get(token, len(token2id)- 1)
 seq.append(token_id)
 return seq
class TextDataset(Dataset):
def __init__(self, df, vocab=None, num_max=None, max_seq_len=100,
max_vocab_size=95000):
if num_max is not None:
df = df[:num_max]
self.src_sents = df['question_text'].tolist()
self.qids = df['qid'].values
if vocab is None:
src_counter = build_counter(self.src_sents)
 vocab = build_vocab(src_counter, max_vocab_size)
 self.vocab = vocab
if 'src_seqs' not in df.columns:
self.src_seqs = []
for sent in tqdm(self.src_sents, desc='tokenize'):
seq = tokens2ids(sent.split() [:max_seq_len], vocab['token2id'])
 self.src_seqs.append(seq)
 else:
self.src_seqs = df['src_seqs'].tolist()
if 'target' in df.columns:
self.targets = df['target'].values
else:
self.targets = np.random.randint(2, size=(len(self.src_sents),))
self.max_seq_len = max_seq_len
def __len__(self):
return len(self.src_sents)
 def get_keys(self):
lens = np.fromiter(
 tqdm(((min(self.max_seq_len, len(c.split())))for c in self.src_sents), desc='generate lens'),
dtype=np.int32)
 return lens
def __getitem__(self, index):
return self.qids[index], self.src_sents[index], self.src_seqs[index], self.targets[index]<categorify>
n_estimators = [10, 100, 500, 1000, 2000]
max_depth = [5, 10, 20]
parameters = dict(n_estimators=n_estimators, max_depth=max_depth)
 rf = RandomForestClassifier(random_state=42)
 rf_grid = GridSearchCV(rf,param_grid=parameters, n_jobs=-1)
 rf_grid_results = perform_model(rf_grid, X_train, y_train, X_test, y_test, class_labels=labels)
 print_grid_search_attributes(rf_grid_results['model'] )
def _pad_sequences(seqs):
lens = [len(seq)for seq in seqs]
max_len = max(lens)
 padded_seqs = torch.zeros(len(seqs), max_len ).long()
for i, seq in enumerate(seqs):
end = lens[i]
padded_seqs[i, :end] = torch.LongTensor(seq)
 return padded_seqs, lens
def collate_fn(data):
qids, src_sents, src_seqs, targets, = zip(*data)
 src_seqs, src_lens = _pad_sequences(src_seqs)
 return qids, src_sents, src_seqs, src_lens, torch.FloatTensor(targets)
 def divide_chunks(l, n):
if n == len(l):
yield np.arange(len(l), dtype=np.int32), l
else:
for i in range(0, len(l), n):
data = l[i:i + n]
yield np.arange(i, i + len(data), dtype=np.int32), data
def prepare_buckets(lens, bucket_size, batch_size, shuffle_data=True, indices=None):
lens = -lens
assert bucket_size % batch_size == 0 or bucket_size == len(lens)
 if indices is None:
if shuffle_data:
indices = shuffle(np.arange(len(lens), dtype=np.int32))
lens = lens[indices]
else:
indices = np.arange(len(lens), dtype=np.int32)
 new_indices = []
extra_batch = None
for chunk_index, chunk in(divide_chunks(lens, bucket_size)) :
indices_sorted = chunk_index[np.argsort(chunk, axis=-1)]
batches = []
for _, batch in divide_chunks(indices_sorted, batch_size):
if len(batch)== batch_size:
batches.append(batch.tolist())
 else:
assert extra_batch is None
assert batch is not None
extra_batch = batch
if shuffle_data:
batches = shuffle(batches)
 for batch in batches:
new_indices.extend(batch)
 if extra_batch is not None:
new_indices.extend(extra_batch)
 return indices[new_indices]
class BucketSampler(Sampler):
def __init__(self, data_source, sort_keys, bucket_size=None, batch_size=1536, shuffle_data=True):
super().__init__(data_source)
 self.shuffle = shuffle_data
self.batch_size = batch_size
self.sort_keys = sort_keys
self.bucket_size = bucket_size if bucket_size is not None else len(sort_keys)
 if not shuffle_data:
self.index = prepare_buckets(self.sort_keys, bucket_size=self.bucket_size, batch_size=self.batch_size,
shuffle_data=self.shuffle)
 else:
self.index = None
self.weights = None
def set_weights(self, w):
assert w >= 0
total = np.sum(w)
 if total != 1:
w = w / total
self.weights = w
def __iter__(self):
indices = None
if self.weights is not None:
total = len(self.sort_keys)
 indices = np.random.choice(total,(total,), p=self.weights)
 if self.shuffle:
self.index = prepare_buckets(self.sort_keys, bucket_size=self.bucket_size, batch_size=self.batch_size,
shuffle_data=self.shuffle, indices=indices)
 return iter(self.index)
 def get_reverse_indexes(self):
indexes = np.zeros(( len(self.index),), dtype=np.int32)
 for i, j in enumerate(self.index):
indexes[j] = i
return indexes
def __len__(self):
return len(self.sort_keys)
<load_pretrained>
model_rf_final = RandomForestClassifier(max_depth= 5, n_estimators= 500)
 model_rf_final.fit(X_train, y_train )
def read_embedding(embedding_file):

 if os.path.basename(embedding_file)!= 'wiki-news-300d-1M.vec':
skip_head = None
else:
skip_head = 0
if os.path.basename(embedding_file)== 'paragram_300_sl999.txt':
encoding = 'latin'
else:
encoding = 'utf-8'
embeddings_index = {}
t_chunks = pd.read_csv(embedding_file, index_col=0, skiprows=skip_head, encoding=encoding, sep=' ', header=None,
quoting=3,
doublequote=False, quotechar=None, engine='c', na_filter=False, low_memory=True,
chunksize=10000)
 for t in t_chunks:
for k, v in zip(t.index.values, t.values):
embeddings_index[k] = v.astype(np.float32)
 return embeddings_index
def get_emb(embedding_index, word, word_raw):
if word == word_raw:
return None
else:
return embedding_index.get(word, None)
 def embedding2numpy(embedding_path, word_index, num_words, embed_size, emb_mean=0., emb_std=0.5,
report_stats=False):
embedding_index = read_embedding(embedding_path)
 num_words = min(num_words + 2, len(word_index))
if report_stats:
all_coefs = []
for v in embedding_index.values() :
all_coefs.append(v.reshape([-1, 1]))
all_coefs = np.concatenate(all_coefs)
 print(all_coefs.mean() , all_coefs.std() , np.linalg.norm(all_coefs, axis=-1 ).mean())
 embedding_matrix = np.zeros(( num_words, embed_size), dtype=np.float32)
 oov = 0
oov_cap = 0
oov_upper = 0
oov_lower = 0
for word, i in word_index.items() :
if i == 0:
continue
if i >= num_words:
continue
embedding_vector = embedding_index.get(word, None)
 if embedding_vector is None:
embedding_vector = get_emb(embedding_index, word.lower() , word)
 if embedding_vector is None:
embedding_vector = get_emb(embedding_index, word.upper() , word)
 if embedding_vector is None:
embedding_vector = get_emb(embedding_index, word.capitalize() , word)
 if embedding_vector is None:
oov += 1
embedding_vector = np.random.normal(emb_mean, emb_std, size=(1, embed_size))
else:
oov_lower += 1
else:
oov_upper += 1
else:
oov_cap += 1
embedding_matrix[i] = embedding_vector
print('oov %d/%d/%d/%d/%d' %(oov, oov_cap, oov_upper, oov_lower, len(word_index)))
 return embedding_matrix
def load_embedding(vocab, max_vocab_size, embed_size):
embedding_matrix1 = embedding2numpy(embedding_glove, vocab['token2id'], max_vocab_size, embed_size,
emb_mean=-0.005838499, emb_std=0.48782197, report_stats=False)
 embedding_matrix2 = embedding2numpy(embedding_para, vocab['token2id'], max_vocab_size, embed_size,
emb_mean=-0.0053247833, emb_std=0.49346462, report_stats=False)
 return [embedding_matrix1, embedding_matrix2] 
<set_options>
test_pred = pd.Series(model_rf_final.predict(test), name = "Survived")
 test_pred_final = pd.DataFrame(test_pred )
def set_lr(optimizer, lr):
for g in optimizer.param_groups:
g['lr'] = lr
class CyclicLR:
def __init__(self, optimizer, base_lr=0.001, max_lr=0.002, step_size=300., mode='triangular',
gamma=0.99994, scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.optimizer = optimizer
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1 /(2.**(x - 1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma ** x
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):
if new_base_lr is not None:
self.base_lr = new_base_lr
if new_max_lr is not None:
self.max_lr = new_max_lr
if new_step_size is not None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1 + self.clr_iterations /(2 * self.step_size))
x = np.abs(self.clr_iterations / self.step_size - 2 * cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr - self.base_lr)* np.maximum(0,(1 - x)) * self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr - self.base_lr)* np.maximum(0,(1 - x)) * self.scale_fn(
 self.clr_iterations)
 def on_train_begin(self):
if self.clr_iterations == 0:
set_lr(self.optimizer, self.base_lr)
 else:
set_lr(self.optimizer, self.clr())
 def on_batch_end(self):
self.trn_iterations += 1
self.clr_iterations += 1
set_lr(self.optimizer, self.clr())
<normalization>
df_gender_submission = pd.read_csv('.. /input/titanic/gender_submission.csv')
class Capsule(nn.Module):
def __init__(self, input_dim_capsule=1024, num_capsule=5, dim_capsule=5, routings=4):
super(Capsule, self ).__init__()
self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.activation = self.squash
self.W = nn.Parameter(
 nn.init.xavier_normal_(torch.empty(1, input_dim_capsule, self.num_capsule * self.dim_capsule)))
 def forward(self, x):
u_hat_vecs = torch.matmul(x, self.W)
 batch_size = x.size(0)
 input_num_capsule = x.size(1)
 u_hat_vecs = u_hat_vecs.view(( batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = u_hat_vecs.permute(0, 2, 1,
3 ).contiguous()
with torch.no_grad() :
b = torch.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
c = torch.nn.functional.softmax(b, dim=1)
 outputs = self.activation(torch.sum(c.unsqueeze(-1)* u_hat_vecs, dim=2))
if i < self.routings - 1:
b =(torch.sum(outputs.unsqueeze(2)* u_hat_vecs, dim=-1))
return outputs
def squash(self, x, axis=-1):
s_squared_norm =(x ** 2 ).sum(axis, keepdim=True)
 scale = torch.sqrt(s_squared_norm + 1e-7)
 return x / scale
class Attention(nn.Module):
def __init__(self, feature_dim, max_seq_len=70):
super().__init__()
self.attention_fc = nn.Linear(feature_dim, 1)
 self.bias = nn.Parameter(torch.zeros(1, max_seq_len, 1, requires_grad=True))
def forward(self, rnn_output):

 attention_weights = self.attention_fc(rnn_output)
 seq_len = rnn_output.size(1)
 attention_weights = self.bias[:, :seq_len, :] + attention_weights
attention_weights = torch.tanh(attention_weights)
 attention_weights = torch.exp(attention_weights)
 attention_weights_sum = torch.sum(attention_weights, dim=1, keepdim=True)+ 1e-7
attention_weights = attention_weights / attention_weights_sum
attended = torch.sum(attention_weights * rnn_output, dim=1)
 return attended
class InsincereModel(nn.Module):
def __init__(self, device, hidden_dim, hidden_dim_fc, embedding_matrixs, vocab_size=None, embedding_dim=None,
dropout=0.1, num_capsule=5, dim_capsule=5, capsule_out_dim=1, alpha=0.8, beta=0.8,
finetuning_vocab_size=120002,
embedding_mode='mixup', max_seq_len=70):
super(InsincereModel, self ).__init__()
self.beta = beta
self.embedding_mode = embedding_mode
self.finetuning_vocab_size = finetuning_vocab_size
self.alpha = alpha
vocab_size, embedding_dim = embedding_matrixs[0].shape
self.raw_embedding_weights = embedding_matrixs
self.embedding_0 = nn.Embedding(vocab_size, embedding_dim, padding_idx=0 ).from_pretrained(
 torch.from_numpy(embedding_matrixs[0]))
self.embedding_1 = nn.Embedding(vocab_size, embedding_dim, padding_idx=0 ).from_pretrained(
 torch.from_numpy(embedding_matrixs[1]))
self.embedding_mean = nn.Embedding(vocab_size, embedding_dim, padding_idx=0 ).from_pretrained(
 torch.from_numpy(( embedding_matrixs[0] + embedding_matrixs[1])/ 2))
self.learnable_embedding = nn.Embedding(finetuning_vocab_size, embedding_dim, padding_idx=0)
 nn.init.constant_(self.learnable_embedding.weight, 0)
 self.learn_embedding = False
self.spatial_dropout = nn.Dropout2d(p=0.2)
 self.device = device
self.hidden_dim = hidden_dim
self.rnn0 = nn.LSTM(embedding_dim, int(hidden_dim / 2), num_layers=1, bidirectional=True, batch_first=True)
 self.rnn1 = nn.GRU(hidden_dim, int(hidden_dim / 2), num_layers=1, bidirectional=True, batch_first=True)
 self.capsule = Capsule(input_dim_capsule=self.hidden_dim, num_capsule=num_capsule, dim_capsule=dim_capsule)
 self.dropout2 = nn.Dropout(0.3)
 self.lincaps = nn.Linear(num_capsule * dim_capsule, capsule_out_dim)
 self.attention1 = Attention(self.hidden_dim, max_seq_len=max_seq_len)
 self.attention2 = Attention(self.hidden_dim, max_seq_len=max_seq_len)
 self.fc = nn.Linear(hidden_dim * 4 + capsule_out_dim, hidden_dim_fc)
 self.norm = torch.nn.LayerNorm(hidden_dim * 4 + capsule_out_dim)
 self.dropout1 = nn.Dropout(0.2)
 self.dropout_linear = nn.Dropout(p=dropout)
 self.hidden2out = nn.Linear(hidden_dim_fc, 1)
 def set_embedding_mode(self, embedding_mode):
self.embedding_mode = embedding_mode
def enable_learning_embedding(self):
self.learn_embedding = True
def init_weights(self):
ih =(param.data for name, param in self.named_parameters() if 'weight_ih' in name)
 hh =(param.data for name, param in self.named_parameters() if 'weight_hh' in name)
 b =(param.data for name, param in self.named_parameters() if 'bias' in name)
 for k in ih:
nn.init.xavier_uniform_(k)
 for k in hh:
nn.init.orthogonal_(k)
 for k in b:
nn.init.constant_(k, 0)
 def apply_spatial_dropout(self, emb):
emb = emb.permute(0, 2, 1 ).unsqueeze(-1)
 emb = self.spatial_dropout(emb ).squeeze(-1 ).permute(0, 2, 1)
 return emb
def forward(self, seqs, lens, return_logits=True):
if self.embedding_mode == 'mixup':
emb0 = self.embedding_0(seqs)
 emb1 = self.embedding_1(seqs)
 prob = np.random.beta(self.alpha, self.beta, size=(seqs.size(0), 1, 1)).astype(np.float32)
 prob = torch.from_numpy(prob ).to(self.device)
 emb = emb0 * prob + emb1 *(1 - prob)
 elif self.embedding_mode == 'emb0':
emb = self.embedding_0(seqs)
 elif self.embedding_mode == 'emb1':
emb = self.embedding_1(seqs)
 elif self.embedding_mode == 'mean':
emb = self.embedding_mean(seqs)
 else:
assert False
if self.learn_embedding:
seq_clamped = torch.clamp(seqs, 0, self.finetuning_vocab_size - 1)
 emb_learned = self.learnable_embedding(seq_clamped)
 emb = emb + emb_learned
emb = self.apply_spatial_dropout(emb)
 lstm_output0, _ = self.rnn0(emb)
 lstm_output1, _ = self.rnn1(lstm_output0)
 content3 = self.capsule(lstm_output1)
 batch_size = content3.size(0)
 content3 = content3.view(batch_size, -1)
 content3 = self.dropout2(content3)
 content3 = torch.relu(self.lincaps(content3))
feature_att1 = self.attention1(lstm_output0)
 feature_att2 = self.attention2(lstm_output1)
 feature_avg2 = torch.mean(lstm_output1, dim=1)
 feature_max2, _ = torch.max(lstm_output1, dim=1)
 feature = torch.cat(( feature_att1, feature_att2, feature_avg2, feature_max2, content3), dim=-1)
 feature = self.norm(feature)
 feature = self.dropout1(feature)
 feature = torch.relu(feature)
 out = self.fc(feature)
 out = self.dropout_linear(out)
 out = self.hidden2out(out)
 if not return_logits:
out = torch.sigmoid(out)
 return out 
<predict_on_test>
submission = pd.DataFrame({
"PassengerId": testo["PassengerId"],
"Survived": test_pred_final['Survived']
})
 submission.to_csv('Titanic Submission.csv', index = False)
 print('Done' )