Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<feature_engineering>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
<define_variables>
train_init_df=pd.read_csv('/kaggle/input/titanic/train.csv')
 train_init_df.head()
mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}<categorify>
test_init_df=pd.read_csv('/kaggle/input/titanic/test.csv')
 test_PassengerId = test_init_df["PassengerId"]
test_init_df.head()
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x<feature_engineering>
train_init_df.isnull().sum()
df['question_text'] = df['question_text'].apply(lambda x: correct_spelling(x, mispell_dict))<feature_engineering>
train_init_df=train_init_df.drop(['Cabin','PassengerId'],axis=1 )
df['size'] = df['question_text'].str.len()
print(mean(df['size']))
print(median(df['size']))
print(stdev(df['size']))
print(amax(df['size']))
print(amin(df['size']))
df = df.drop(['size'],axis=1 )<prepare_x_and_y>
train_init_df[["Pclass","Survived"]].groupby(["Pclass"], as_index = False ).mean().sort_values(by = "Survived", ascending = False )
total_set_qid = df['qid'].values
total_set_X = df["question_text"].fillna("_na_" ).values
total_set_y = df['target'].values<string_transform>
train_init_df[["Sex","Survived"]].groupby(["Sex"], as_index = False ).mean().sort_values(by = "Survived", ascending = False )
tokenizer = Tokenizer(num_words=None, filters='')
 tokenizer.fit_on_texts(list(total_set_X))
total_set_X = tokenizer.texts_to_sequences(total_set_X )<string_transform>
train_init_df[["SibSp","Survived"]].groupby(["SibSp"], as_index = False ).mean().sort_values(by = "Survived", ascending = False )
total_set_X = pad_sequences(total_set_X, maxlen=question_length)
<create_dataframe>
train_init_df[["Parch","Survived"]].groupby(["Parch"], as_index = False ).mean().sort_values(by = "Survived", ascending = False )
df = pd.concat([pd.DataFrame(total_set_qid),pd.DataFrame(total_set_X), pd.DataFrame(total_set_y)], axis=1, keys=["qid", "question_text", "target"] )<categorify>
train_init_df[["Embarked","Survived"]].groupby(["Embarked"], as_index = False ).mean().sort_values(by = "Survived", ascending = False )
def embedding_matrix_creator(embeddings_index):
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = len(word_index)
 embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words+1, embed_size))
for word, i in word_index.items() :
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<categorify>
def checkNameSalutations(name):
salutation = re.findall(r'(\w{2,})\.',name)
 return salutation[0]
emb_matrix = embedding_matrix_creator(my_embedding_matrix )<drop_column>
train_init_df['Salutation']=train_init_df['Name'].apply(checkNameSalutations)
 train_init_df.head()
<set_options>
train_init_df[["Salutation","Survived"]].groupby(["Salutation"], as_index = False ).mean().sort_values(by = "Survived", ascending = False )
<split>
def binaryGender(val):
if val=='male':
return 0
elif val=='female':
return 1
train_df, val_df = train_test_split(df[df.target[0]!=-1], test_size=0.1)
 train_X = np.array(train_df["question_text"])
 train_y = np.array(train_df["target"])
 val_X = np.array(val_df["question_text"])
 val_y = np.array(val_df["target"] )<drop_column>
train_init_df['Sex']= train_init_df['Sex'].apply(binaryGender)
 train_init_df.head()
test_df = df[df.target[0]==-1]
test_X=np.array(test_df["question_text"])
<compute_test_metric>
train_init_df[["Sex","Survived"]].groupby(["Sex"], as_index = False ).mean().sort_values(by = "Survived", ascending = False )
def f1(y_true, y_pred):
def recall(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))<set_options>
def checkAplphabetInTicket(ticket):
alpha=re.findall(r'(\w{1,}\D)',ticket)
 if len(alpha)!=0:
return 1
else:
return 0
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
train_init_df['Alpha_ticket']= train_init_df['Ticket'].apply(checkAplphabetInTicket)
 train_init_df.head()
def make_old_model(embedding_matrix, embed_size=300, loss='binary_crossentropy'):
inp = Input(shape=(question_length,))
x = Embedding(embedding_matrix.shape[0], embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 avg_pl = GlobalAveragePooling1D()(x)
 max_pl = GlobalMaxPooling1D()(x)
 concat = concatenate([avg_pl, max_pl])
 dense = Dense(64, activation="relu" )(concat)
 drop = Dropout(0.1 )(concat)
 output = Dense(1, activation="sigmoid" )(concat)
 model = Model(inputs=inp, outputs=output)
 model.compile(loss=loss, optimizer=Adam(lr=0.0001), metrics=['accuracy', f1])
 return model<define_search_model>
train_init_df['Alpha_ticket'].value_counts()
def model_lstm_gru_atten(embedding_matrix, embed_size=300, loss='binary_crossentropy'):
inp = Input(shape=(question_length,))
x = Embedding(embedding_matrix.shape[0], embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.1,seed=seed_nb )(x)
 x = Bidirectional(CuDNNLSTM(64, kernel_initializer=glorot_uniform(seed=seed_nb), return_sequences=True))(x)
 y = Bidirectional(CuDNNGRU(40,kernel_initializer=glorot_uniform(seed=seed_nb), return_sequences=True))(x)
 atten_1 = Attention(question_length )(x)
 atten_2 = Attention(question_length )(y)
 avg_pool = GlobalAveragePooling1D()(y)
 max_pool = GlobalMaxPooling1D()(y)
 conc = concatenate([atten_1, atten_2, avg_pool, max_pool])
 conc = Dense(16,kernel_initializer=he_uniform(seed=seed_nb), activation="relu" )(conc)
 conc = Dropout(0.1,seed=seed_nb )(conc)
 outp = Dense(1,kernel_initializer=he_uniform(seed=seed_nb), activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss=loss, optimizer=Adam(lr=0.0001), metrics=['accuracy', f1])
 return model<choose_model_class>
train_init_df[["Alpha_ticket","Survived"]].groupby(["Alpha_ticket"], as_index = False ).mean().sort_values(by = "Survived", ascending = False )
checkpoints = ModelCheckpoint('weights.hdf5', monitor="val_f1", mode="max", verbose=True, save_best_only=True)
 reduce_lr = ReduceLROnPlateau(monitor='val_f1', factor=0.1, patience=2, verbose=1, min_lr=0.000001 )<compute_test_metric>
train_init_df["Salutation"] = train_init_df["Salutation"].replace(["Lady","Countess","Capt","Col","Don","Dr","Major","Rev","Sir","Jonkheer","Dona"],"other")
 train_init_df["Salutation"] = [0 if i == "Master" else 1 if i == "Miss" or i == "Ms" or i == "Mlle" or i == "Mrs" else 2 if i == "Mr" else 3 for i in train_init_df["Salutation"]]
train_init_df["Salutation"].head(20 )
def tweak_threshold(pred, truth):
thresholds = []
scores = []
print("Threshold: Valor")
 for thresh in np.arange(0.01, 1.01, 0.01):
thresh = np.round(thresh, 2)
 thresholds.append(thresh)
 score = f1_score(truth,(pred>thresh ).astype(int))
print(thresh, ": ", score)
 scores.append(score)
 return np.max(scores), thresholds[np.argmax(scores)]<split>
train_init_df.drop(labels= ["Name"], axis = 1, inplace = True)
df_X = df[df.target[0]!=-1]["question_text"]
df_y = df[df.target[0]!=-1]["target"]
train_meta = np.zeros(df_y.shape)
 test_meta = np.zeros(test_X.shape[0])
 my_splits = list(StratifiedKFold(n_splits=K_FOLDS,
shuffle=True,
random_state=DATA_SPLIT_SEED ).split(df_X, df_y))
for idx,(train_idx, valid_idx)in enumerate(my_splits):
print("======== K-FOLD: {0} ==========".format(idx))
train_X = df_X.iloc[train_idx]
train_y = df_y.iloc[train_idx]
val_X = df_X.iloc[valid_idx]
val_y = df_y.iloc[valid_idx]
model = make_old_model(emb_matrix)
 history = model.fit(train_X, train_y, batch_size=batch_size, epochs=K_FOLD_EPOCHS, validation_data=[val_X, val_y], callbacks=[checkpoints, reduce_lr])
 model.load_weights('weights.hdf5')
 eval_pred = model.predict(val_X, batch_size=batch_size, verbose=1)
 test_pred = model.predict(test_X, batch_size=batch_size, verbose=1)
 train_meta[valid_idx] = eval_pred
test_meta += test_pred.reshape(-1)/ len(my_splits )<compute_test_metric>
train_init_df.drop(labels= ["Ticket"], axis = 1, inplace = True )
score_val, best_thresh = tweak_threshold(train_meta, df_y)
 print("=====================================")
 print(f"Scored {round(score_val, 4)} for threshold {best_thresh} with untreated texts on validation data" )<save_to_csv>
train_init_df.isnull().sum()
y_te =(np.array(test_meta)> best_thresh ).astype(np.int)
 qid = test_df["qid"].values
submit_df = pd.concat([pd.DataFrame(qid, columns=['qid']),pd.DataFrame(y_te, columns=['prediction'])], axis = 1)
 submit_df.to_csv("submission_cesc.csv", index=False )<set_options>
train_init_df["Embarked"] = train_init_df["Embarked"].fillna("C")
 train_init_df[train_init_df["Embarked"].isnull() ]
%matplotlib inline
pd.set_option('max_colwidth',400)
 warnings.filterwarnings("ignore", message="F-score is ill-defined and being set to 0.0 due to no predicted samples.")
<set_options>
index_nan_age = list(train_init_df["Age"][train_init_df["Age"].isnull() ].index)
 for i in index_nan_age:
age_pred = train_init_df["Age"][(( train_init_df["SibSp"] == train_init_df.iloc[i]["SibSp"])&(train_init_df["Parch"] == train_init_df.iloc[i]["Parch"])&(train_init_df["Pclass"] == train_init_df.iloc[i]["Pclass"])) ].median()
age_med = train_init_df["Age"].median()
if not np.isnan(age_pred):
train_init_df["Age"].iloc[i] = age_pred
else:
train_init_df["Age"].iloc[i] = age_med
def seed_torch(seed=1029):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True<load_from_csv>
train_init_df = pd.get_dummies(train_init_df,columns = ["Embarked"])
 train_init_df.head()
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 sub = pd.read_csv('.. /input/sample_submission.csv' )<define_variables>
train_init_df = pd.get_dummies(train_init_df,columns = ["Salutation"])
 train_init_df.head()
sin = len(train[train["target"] == 0])
 insin = len(train[train["target"] == 1])
 tot = sin + insin
print("Train sincere text total {}".format(sin))
print("Train insincere text total {}".format(insin))
print("Sincere percentage = {0:.2f}".format(sin*100/tot))
print("Sincere percentage = {0:.2f}".format(insin*100/tot))<split>
train_init_df = pd.get_dummies(train_init_df,columns = ["Pclass"])
 train_init_df.head()
print('Average word length of questions in train is {0:.0f}.'.format(np.mean(train['question_text'].apply(lambda x: len(x.split())))))
 print('Average word length of questions in test is {0:.0f}.'.format(np.mean(test['question_text'].apply(lambda x: len(x.split())))) )<string_transform>
train_init_df = pd.get_dummies(train_init_df,columns = ["Alpha_ticket"])
 train_init_df.head()
print('Max word length of questions in train is {0:.0f}.'.format(np.max(train['question_text'].apply(lambda x: len(x.split())))))
 print('Max word length of questions in test is {0:.0f}.'.format(np.max(test['question_text'].apply(lambda x: len(x.split())))) )<compute_test_metric>
train_init_df["Sex"] = train_init_df["Sex"].astype("category")
 train_init_df = pd.get_dummies(train_init_df, columns = ["Sex"])
 train_init_df.head()
print('Average character length of questions in train is {0:.0f}.'.format(np.mean(train['question_text'].apply(lambda x: len(x)))))
 print('Average character length of questions in test is {0:.0f}.'.format(np.mean(test['question_text'].apply(lambda x: len(x)))) )<define_variables>
train_init_df["Fsize"] = train_init_df["SibSp"] + train_init_df["Parch"] + 1
train_init_df.head()
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"that's" : "that is",
"there's" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not",
"tryin'":"trying"}
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
 train["question_text"] = train["question_text"].apply(lambda x: clean_text(x.lower()))
test["question_text"] = test["question_text"].apply(lambda x: clean_text(x.lower()))
train["question_text"] = train["question_text"].apply(lambda x: clean_numbers(x))
test["question_text"] = test["question_text"].apply(lambda x: clean_numbers(x))
train["question_text"] = train["question_text"].apply(lambda x: replace_typical_misspell(x))
test["question_text"] = test["question_text"].apply(lambda x: replace_typical_misspell(x))<train_model>
train_init_df["family_size"] = [1 if i < 5 else 0 for i in train_init_df["Fsize"]]
train_init_df.head()
max_features = 120000
tk = Tokenizer(lower = True, filters='', num_words=max_features)
 full_text = list(train['question_text'].values)+ list(test['question_text'].values)
 tk.fit_on_texts(full_text )<string_transform>
train_init_df = pd.get_dummies(train_init_df, columns= ["family_size"])
 train_init_df.head()
train_tokenized = tk.texts_to_sequences(train['question_text'].fillna('_
test_tokenized = tk.texts_to_sequences(test['question_text'].fillna('_<categorify>
test_df = train_init_df.copy(deep=True)
 test_df.drop(labels = ["Survived"],axis = 1, inplace = True )
max_len = 72
maxlen = 72
X_train = pad_sequences(train_tokenized, maxlen = max_len)
 X_test = pad_sequences(test_tokenized, maxlen = max_len )<prepare_x_and_y>
train = train_init_df.copy(deep=True)
 X_train = train.drop(labels = "Survived", axis = 1)
 y_train = train["Survived"]
X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, test_size = 0.33, random_state = 42)
 print("X_train",len(X_train))
print("X_test",len(X_test))
print("y_train",len(y_train))
print("y_test",len(y_test))
print("test",len(test_df))
y_train = train['target'].values<compute_test_metric>
logreg = LogisticRegression()
logreg.fit(X_train, y_train)
 acc_log_train = round(logreg.score(X_train, y_train)*100,2)
 acc_log_test = round(logreg.score(X_test, y_test)*100,2)
 print("Training Accuracy: % {}".format(acc_log_train))
print("Testing Accuracy: % {}".format(acc_log_test))
def sigmoid(x):
return 1 /(1 + np.exp(-x))<split>
random_state = 42
classifier = [DecisionTreeClassifier(random_state = random_state),
SVC(random_state = random_state),
RandomForestClassifier(random_state = random_state),
LogisticRegression(random_state = random_state),
KNeighborsClassifier() ]
dt_param_grid = {"min_samples_split" : range(10,500,20),
"max_depth": range(1,20,2)}
svc_param_grid = {"kernel" : ["rbf"],
"gamma": [0.001, 0.01, 0.1, 1],
"C": [1,10,50,100,200,300,1000]}
rf_param_grid = {"max_features": [1,3,10],
"min_samples_split":[2,3,10],
"min_samples_leaf":[1,3,10],
"bootstrap":[False],
"n_estimators":[100,300],
"criterion":["gini"]}
logreg_param_grid = {"C":np.logspace(-3,3,7),
"penalty": ["l1","l2"]}
knn_param_grid = {"n_neighbors": np.linspace(1,19,10, dtype = int ).tolist() ,
"weights": ["uniform","distance"],
"metric":["euclidean","manhattan"]}
classifier_param = [dt_param_grid,
svc_param_grid,
rf_param_grid,
logreg_param_grid,
knn_param_grid]
splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=10 ).split(X_train, y_train))<statistical_test>
cv_result = []
best_estimators = []
for i in range(len(classifier)) :
clf = GridSearchCV(classifier[i], param_grid=classifier_param[i], cv = StratifiedKFold(n_splits = 10), scoring = "accuracy", n_jobs = -1,verbose = 1)
 clf.fit(X_train,y_train)
 cv_result.append(clf.best_score_)
 best_estimators.append(clf.best_estimator_)
 print(cv_result[i] )
embed_size = 300
embedding_path = ".. /input/embeddings/glove.840B.300d/glove.840B.300d.txt"
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embedding_index = dict(get_coefs(*o.split(" ")) for o in open(embedding_path, encoding='utf-8', errors='ignore'))
emb_mean,emb_std = -0.005838499, 0.48782197
word_index = tk.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words + 1, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embedding_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<statistical_test>
votingC = VotingClassifier(estimators = [("dt",best_estimators[0]), 
("rfc",best_estimators[2]), 
("lr",best_estimators[3])],
voting = "soft", n_jobs = -1)
 votingC = votingC.fit(X_train, y_train)
 print(accuracy_score(votingC.predict(X_test),y_test))
embedding_path = ".. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt"
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embedding_index = dict(get_coefs(*o.split(" ")) for o in open(embedding_path, encoding='utf-8', errors='ignore')if len(o)>100)
 emb_mean,emb_std = -0.0053247833, 0.49346462
embedding_matrix1 = np.random.normal(emb_mean, emb_std,(nb_words + 1, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embedding_index.get(word)
 if embedding_vector is not None: embedding_matrix1[i] = embedding_vector<compute_test_metric>
test_init_df.drop(['Cabin'],axis=1,inplace=True )
embedding_matrix = np.mean([embedding_matrix, embedding_matrix1], axis=0)
 del embedding_matrix1<normalization>
test_final_sub=test_init_df['PassengerId']
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1)
 class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self ).__init__()
hidden_size = 128
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.1)
 self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm_attention = Attention(hidden_size*2, maxlen)
 self.gru_attention = Attention(hidden_size*2, maxlen)
 self.linear = nn.Linear(512, 16)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.out = nn.Linear(16, 1)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 h_lstm_atten = self.lstm_attention(h_lstm)
 h_gru_atten = self.gru_attention(h_gru)
 conc = torch.cat(( h_lstm_atten, h_gru_atten), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out<choose_model_class>
test_init_df['Salutation']=test_init_df['Name'].apply(checkNameSalutations)
 test_init_df.head()
m = NeuralNet()
print(m )<train_model>
test_init_df["Salutation"] = test_init_df["Salutation"].replace(["Lady","Countess","Capt","Col","Don","Dr","Major","Rev","Sir","Jonkheer","Dona"],"other")
 test_init_df["Salutation"] = [0 if i == "Master" else 1 if i == "Miss" or i == "Ms" or i == "Mlle" or i == "Mrs" else 2 if i == "Mr" else 3 for i in test_init_df["Salutation"]]
test_init_df["Salutation"].head(20 )
def train_model(model, x_train, y_train, x_val, y_val, validate=True):
optimizer = torch.optim.Adam(model.parameters())
 train = torch.utils.data.TensorDataset(x_train, y_train)
 valid = torch.utils.data.TensorDataset(x_val, y_val)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 loss_fn = torch.nn.BCEWithLogitsLoss(reduction='mean' ).cuda()
best_score = -np.inf
for epoch in range(n_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for x_batch, y_batch in tqdm(train_loader, disable=True):
y_pred = model(x_batch)
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds = np.zeros(( x_val_fold.size(0)))
 if validate:
avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
search_result = threshold_search(y_val.cpu().numpy() , valid_preds)
 val_f1, val_threshold = search_result['f1'], search_result['threshold']
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t val_f1={:.4f} best_t={:.2f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, avg_val_loss, val_f1, val_threshold, elapsed_time))
else:
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, elapsed_time))
valid_preds = np.zeros(( x_val_fold.size(0)))
 avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
print('Validation loss: ', avg_val_loss)
 test_preds = np.zeros(( len(test_loader.dataset)))
 for i,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch ).detach()
test_preds[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
return valid_preds, test_preds<create_dataframe>
test_init_df.isnull().sum()
x_test_cuda = torch.tensor(X_test, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 batch_size = 512
test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False )<compute_train_metric>
test_init_df.drop(['Name'],axis=1,inplace=True )
seed=1029
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)], disable=True):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<prepare_x_and_y>
test_init_df['Sex']= test_init_df['Sex'].apply(binaryGender)
 test_init_df.head()
train_preds = np.zeros(len(train))
test_preds = np.zeros(( len(test), len(splits)))
 n_epochs = 5
for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(X_train[train_idx], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(y_train[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(X_train[valid_idx], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(y_train[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()
train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 seed_everything(seed + i)
 model = NeuralNet()
model.cuda()
valid_preds_fold, test_preds_fold = train_model(model,
x_train_fold,
y_train_fold,
x_val_fold,
y_val_fold, validate=False)
 train_preds[valid_idx] = valid_preds_fold
test_preds[:, i] = test_preds_fold 
<save_to_csv>
test_init_df['Alpha_ticket']= test_init_df['Ticket'].apply(checkAplphabetInTicket)
 test_init_df.head()
search_result = threshold_search(y_train, train_preds)
 print(search_result)
 sub['prediction'] = test_preds.mean(1)> search_result['threshold']
sub.to_csv("submission.csv", index=False )<import_modules>
test_init_df['Alpha_ticket'].value_counts()
pd.set_option("display.max_colwidth", 200)
 sns.set_style('darkgrid')
 pd.options.display.float_format = '{:,.3f}'.format<load_from_csv>
test_init_df.drop(labels= ["Ticket"], axis = 1, inplace = True)
 test_init_df.head()
%%time
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print(train.shape, test.shape )<drop_column>
index_nan_age = list(test_init_df["Age"][test_init_df["Age"].isnull() ].index)
 for i in index_nan_age:
age_pred = test_init_df["Age"][(( test_init_df["SibSp"] == test_init_df.iloc[i]["SibSp"])&(test_init_df["Parch"] == test_init_df.iloc[i]["Parch"])&(test_init_df["Pclass"] == test_init_df.iloc[i]["Pclass"])) ].median()
age_med = test_init_df["Age"].median()
if not np.isnan(age_pred):
test_init_df["Age"].iloc[i] = age_pred
else:
test_init_df["Age"].iloc[i] = age_med
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )<normalization>
test_init_df[test_init_df["Fare"].isnull() ]
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }
ZEN = "".join(chr(0xff01 + i)for i in range(94))
HAN = "".join(chr(0x21 + i)for i in range(94))
ZEN2HAN = str.maketrans(ZEN, HAN)
 def preprocess(data):
def clean_special_chars(text):
punct = "/-'?!.,
for p in punct:
text = text.replace(p, ' ')
 for p in punct_mapping:
text = text.replace(p, punct_mapping[p])
 text = text.translate(ZEN2HAN)
 return text
data = data.astype(str ).apply(lambda x: clean_special_chars(x))
return data
train['question_text'] = preprocess(train['question_text'])
 test['question_text'] = preprocess(test['question_text'] )<concatenate>
test_init_df["Fare"] = test_init_df["Fare"].fillna(np.mean(test_init_df[test_init_df["Pclass"] == 3]["Fare"]))
text_to_word_sequence(train['question_text'].values[0] )<prepare_x_and_y>
test_init_df[test_init_df["Fare"].isnull() ]
X_train = train.drop(['qid','target'], axis=1)
 Y_train = train['target']
X_test = test.drop(['qid'], axis=1)
 del train, test
print(X_train.shape, X_test.shape )<load_pretrained>
test_init_df = pd.get_dummies(test_init_df,columns = ["Embarked"])
 test_init_df.head()
TARGET_COLUMN = 'target'
TEXT_COLUMN = 'question_text'
MAX_NUM_WORDS = 300000
TOKENIZER_FILTER = '\r\t
'
tokenizer = Tokenizer(num_words=MAX_NUM_WORDS, filters=TOKENIZER_FILTER)
 tokenizer.fit_on_texts(list(X_train[TEXT_COLUMN])+ list(X_test[TEXT_COLUMN]))<sort_values>
test_init_df = pd.get_dummies(test_init_df,columns = ["Salutation"])
 test_init_df.head()
tokenizer_tx = Tokenizer(num_words=MAX_NUM_WORDS, filters=TOKENIZER_FILTER)
 tokenizer_tx.fit_on_texts(list(X_train.loc[Y_train == 1, TEXT_COLUMN]))
counter = sorted(dict(tokenizer_tx.word_docs ).items() , key=lambda x:x[1], reverse=True)
 wordcount_tx = pd.Series([x[1] for x in counter], [x[0] for x in counter])
 wordcount_stats = pd.concat([wordcount, wordcount_tx], axis=1, keys=['all', 'toxic'], sort=False)
 wordcount_tx = wordcount_stats[wordcount_stats['all'] * 0.8 <= wordcount_stats['toxic']].copy()
wordcount_tx[wordcount_tx['toxic'] > 0]
wordcount_tx.sort_values(by='toxic', ascending=False, inplace=True)
 print(len(wordcount_tx))
wordcount_tx['toxic'][:30].plot.bar(color='red', width=0.7, figsize=(12,3))<remove_duplicates>
test_init_df = pd.get_dummies(test_init_df,columns = ["Pclass"])
 test_init_df.head()
wordcount = pd.concat([wordcount_tx['all'], wordcount] ).to_frame().reset_index()
wordcount.drop_duplicates(keep='first', inplace=True)
 wordcount = wordcount.set_index('index')[0]
del counter, wordcount_tx, wordcount_stats<define_variables>
test_init_df = pd.get_dummies(test_init_df,columns = ["Alpha_ticket"])
 test_init_df.head()
VOCAB_SIZE = 50000
print('covered until', wordcount[VOCAB_SIZE], 'times word' )<define_variables>
test_init_df["Sex"] = test_init_df["Sex"].astype("category")
 test_init_df = pd.get_dummies(test_init_df, columns = ["Sex"])
 test_init_df.head()
%%time
EMBEDDINGS_DIMENSION = 300
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f)
 def build_matrix(path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( VOCAB_SIZE + 1, EMBEDDINGS_DIMENSION))
unknown_words = []
for i in range(VOCAB_SIZE):
try:
word = wordcount.index[i]
embedding_matrix[i] = embedding_index[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words
glove_matrix, unknown_words_glove = build_matrix(EMBEDDING_FILE)
 word2index = dict(( wordcount.index[i], i)for i in range(VOCAB_SIZE))
embedding_matrix = glove_matrix
embedding_matrix.shape<define_variables>
test_init_df["Fsize"] = test_init_df["SibSp"] + test_init_df["Parch"] + 1
test_init_df.head()
words_count = len(unknown_words_glove)
 print('n unknown words(glove):', words_count, ', {:.3%} of all words'.format(words_count / n_words))
print('unknown words(glove):', unknown_words_glove )<string_transform>
test_init_df["family_size"] = [1 if i < 5 else 0 for i in test_init_df["Fsize"]]
test_init_df.head()
MAX_SEQUENCE_LENGTH = 128
def word_index(word):
try:
return word2index[word]
except KeyError:
return VOCAB_SIZE
def pad_text(texts, tokenizer):
matrix = [list(map(word_index, text_to_word_sequence(t, filters=TOKENIZER_FILTER)))for t in texts]
return pad_sequences(matrix, maxlen=MAX_SEQUENCE_LENGTH)
 train_text = pad_text(X_train[TEXT_COLUMN], tokenizer)
 test_text = pad_text(X_test[TEXT_COLUMN], tokenizer )<create_dataframe>
test_init_df = pd.get_dummies(test_init_df, columns= ["family_size"])
 test_init_df.head()
del(X_train, X_test)
 gc.collect()
print(pd.DataFrame([[val for val in dir() ], [sys.getsizeof(eval(val)) for val in dir() ]],
index=['name','size'] ).T.sort_values('size', ascending=False ).reset_index(drop=True)[:10] )<compute_train_metric>
test_init_df.drop(['PassengerId'],axis=1,inplace=True)
 test_init_df.head()
class F1Callback(Callback):
def __init__(self):
self.f1s = []
def on_epoch_end(self, epoch, logs):
eps = np.finfo(np.float32 ).eps
recall = logs["val_true_positives"] /(logs["val_possible_positives"] + eps)
 precision = logs["val_true_positives"] /(logs["val_predicted_positives"] + eps)
 f1 = 2*precision*recall /(precision+recall+eps)
 print("f1_val(from log)=", f1)
 self.f1s.append(f1)
 def true_positives(y_true, y_pred):
return K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 def possible_positives(y_true, y_pred):
return K.sum(K.round(K.clip(y_true, 0, 1)))
 def predicted_positives(y_true, y_pred):
return K.sum(K.round(K.clip(y_pred, 0, 1)) )<set_options>
test_survived = pd.Series(votingC.predict(test_init_df), name = "Survived" ).astype(int)
 results = pd.concat([test_PassengerId, test_survived],axis = 1)
 results.to_csv("titanic.csv", index = False )