Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_from_csv>
%matplotlib inline
INPUT_PATH=".. /input/"
train = pd.read_csv(INPUT_PATH+'google-quest-challenge/train.csv')
 test = pd.read_csv(INPUT_PATH+'google-quest-challenge/test.csv')
 submission = pd.read_csv(INPUT_PATH+'google-quest-challenge/sample_submission.csv' )<define_variables>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 train.head()
print(train['Embarked'].unique() , train['Pclass'].unique())
 train[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean().sort_values(by='Survived', ascending=False)
targets = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written'
]
input_columns = ['question_title','question_body','answer'] 
<define_variables>
for df in [train, test]:
df.drop(labels=["PassengerId", "Cabin", "Name", "Ticket"], axis=1, inplace=True )
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', '\xa0', '\t',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '\u3000', '\u202f',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '«',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"couldnt" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"doesnt" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"havent" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"shouldnt" : "should not",
"that's" : "that is",
"thats" : "that is",
"there's" : "there is",
"theres" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"theyre": "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not",
"tryin'":"trying"}
def clean_text(x):
x = str(x ).replace("
","")
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
def replace_typical_misspell(text):
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
 def clean_data(df, columns: list):
for col in columns:
df[col] = df[col].apply(lambda x: clean_text(x.lower()))
df[col] = df[col].apply(lambda x: replace_typical_misspell(x))
return df 
<drop_column>
for df in [train, test]:
for col in ["Age", "Fare"]:
df[col] = df[col].fillna(np.mean(df[col]))
train = clean_data(train, input_columns)
 test = clean_data(test, input_columns )<string_transform>
min_max_scaler = preprocessing.MinMaxScaler()
for df in [train, test]:
for col in ["Age", "Fare"]:
x = df[[col]].values.astype(float)
 df[col] = min_max_scaler.fit_transform(x )
def constructLabeledSentences(data):
sentences=[]
for index, row in data.iteritems() :
sentences.append(TaggedDocument(utils.to_unicode(row ).split() , ['Text' + '_%s' % str(index)]))
return sentences
def textClean(text):
text = re.sub(r"[^A-Za-z0-9^,!.\/'+-=]", " ", text)
 text = text.lower().split()
stops = set(stopwords.words("english"))
text = [w for w in text if not w in stops]
text = " ".join(text)
 return(text)
 def cleanup(text):
text = textClean(text)
 text= text.translate(str.maketrans("","", string.punctuation))
return text
train_question_body_sentences = constructLabeledSentences(train['question_body'])
 train_question_title_sentences = constructLabeledSentences(train['question_title'])
 train_answer_sentences = constructLabeledSentences(train['answer'])
 test_question_body_sentences = constructLabeledSentences(test['question_body'])
 test_question_title_sentences = constructLabeledSentences(test['question_title'])
 test_answer_sentences = constructLabeledSentences(test['answer'] )<train_model>
for df in [train, test]:
df['is_male'] = np.where(df['Sex']=="male", 1, 0)
 df['is_female'] = np.where(df['Sex']=="female", 1, 0)
 df['EmbarkedS'] = np.where(df['Embarked']=="S", 1, 0)
 df['EmbarkedC'] = np.where(df['Embarked']=="C", 1, 0)
 df['EmbarkedQ'] = np.where(df['Embarked']=="Q", 1, 0)
 df['Pclass1'] = np.where(df['Pclass']==1, 1, 0)
 df['Pclass2'] = np.where(df['Pclass']==2, 1, 0)
 df['Pclass3'] = np.where(df['Pclass']==3, 1, 0)
 df['is_single'] = np.where(np.logical_and(df['SibSp']==0, df['Parch']==0), 1, 0)
 for df in [train, test]:
df.drop(labels=["Sex", "Embarked", 'Pclass'], axis=1, inplace=True )
all_sentences = train_question_body_sentences + \
train_answer_sentences + \
test_question_body_sentences + \
test_answer_sentences
Text_INPUT_DIM=128
text_model = Doc2Vec(min_count=1, window=5, vector_size=Text_INPUT_DIM, sample=1e-4, negative=5, workers=4, epochs=5,seed=1)
 text_model.build_vocab(all_sentences)
 text_model.train(all_sentences, total_examples=text_model.corpus_count, epochs=text_model.iter )<string_transform>
train_size = int(train.shape[0] * 0.85)
 train_dataset = train[:train_size]
val_dataset = train[train_size:]
X_train = train_dataset.drop(labels=["Survived"], axis=1 ).values
Y_train = train_dataset["Survived"].values
X_val = val_dataset.drop(labels=["Survived"], axis=1 ).values
Y_val = val_dataset["Survived"].values
input_size = len(train_dataset.columns)- 1
def normalize_sentence(tokens):
lemmatizer = WordNetLemmatizer()
lemmatized_sentence = []
for word, tag in pos_tag(tokens):
if tag.startswith('NN')or tag.startswith('PRP'):
pos = 'n'
elif tag.startswith('VB'):
pos = 'v'
else:
continue
pos = 'a'
lemmatized_sentence.append(lemmatizer.lemmatize(word, pos ).lower())
 return lemmatized_sentence 
<feature_engineering>
model = Sequential()
k_init = 'glorot_uniform'
optimizer = optimizers.Adam()
model.add(Dense(64,input_dim=input_size, kernel_initializer=k_init))
model.add(Activation("relu"))
model.add(Dropout(0.3))
model.add(Dense(64, kernel_initializer=k_init))
model.add(Activation("relu"))
model.add(Dropout(0.3))
model.add(Dense(1, kernel_initializer=k_init))
model.add(Activation("sigmoid"))
model.compile(loss=losses.binary_crossentropy, optimizer=optimizer,
metrics=[metrics.binary_accuracy])
 history = model.fit(X_train, Y_train, validation_data=(X_val, Y_val),
epochs=200, batch_size=100,
verbose=0)
def normalize_vectorize(df, columns: list):
for col in columns:
print(col)
 df[col+'_norm'] = df[col].apply(lambda x: ' '.join(set(normalize_sentence(word_tokenize(x)))))
 df[col+'_vec'] = df[col].apply(lambda x: text_model.infer_vector([x]))
return df
train = normalize_vectorize(train, input_columns)
 test = normalize_vectorize(test, input_columns)
<load_pretrained>
y_final = model.predict_classes(test.values ).reshape(-1)
 df_test = pd.read_csv(".. /input/test.csv")
 output = pd.DataFrame({'PassengerId': df_test['PassengerId'], 'Survived': y_final})
 surv_num = sum(output["Survived"] != 0)/ len(output)
 print(f"Survive ratio: {surv_num}" )
try:
pbe = load_obj(".. /input/questembeddings/precomputed_bert_embeddings")
 train_question_body_dense = pbe['train_question_body_dense']
train_answer_dense = pbe['train_answer_dense']
train_question_title_dense = pbe['train_question_title_dense']
test_question_body_dense = pbe['test_question_body_dense']
test_answer_dense = pbe['test_answer_dense']
test_question_title_dense = pbe['test_question_title_dense']
except:
print("Load failed, build embedding")
 def sigmoid(x):
return 1 /(1 + math.exp(-x))
def chunks(l, n):

 for i in range(0, len(l), n):
yield l[i:i + n]
def fetch_vectors(string_list, batch_size=64):
DEVICE = torch.device("cuda")
 tokenizer = transformers.DistilBertTokenizer.from_pretrained(".. /input/distilbertbaseuncased/")
 model = transformers.DistilBertModel.from_pretrained(".. /input/distilbertbaseuncased/")
 model.to(DEVICE)
 fin_features = []
for data in tqdm_notebook(chunks(string_list, batch_size)) :
tokenized = []
for x in data:
x = " ".join(x.strip().split() [:300])
 tok = tokenizer.encode(x, add_special_tokens=True)
 tokenized.append(tok[:512])
 max_len = 512
padded = np.array([i + [0] *(max_len - len(i)) for i in tokenized], dtype='int64')
 attention_mask = np.where(padded != 0, 1, 0)
 input_ids = torch.tensor(padded ).to(DEVICE)
 attention_mask = torch.tensor(attention_mask ).to(DEVICE)
 with torch.no_grad() :
last_hidden_states = model(input_ids, attention_mask=attention_mask)
 features = last_hidden_states[0][:, 0, :].cpu().numpy()
fin_features.append(features)
 fin_features = np.vstack(fin_features)
 return fin_features
train_question_body_dense = fetch_vectors(train.question_body.values)
 train_answer_dense = fetch_vectors(train.answer.values)
 train_question_title_dense = fetch_vectors(train.question_title.values)
 test_question_body_dense = fetch_vectors(test.question_body.values)
 test_answer_dense = fetch_vectors(test.answer.values)
 test_question_title_dense = fetch_vectors(test.question_title.values)
 precomputed_bert_embeddings = {
'train_question_body_dense': train_question_body_dense,
'train_answer_dense': train_answer_dense,
'train_question_title_dense': train_question_title_dense,
'test_question_body_dense': test_question_body_dense,
'test_answer_dense': test_answer_dense,
'test_question_title_dense': test_question_title_dense,
} 
<categorify>
output.to_csv('prediction-ann.csv', index=False)
 output