Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<categorify>
%matplotlib inline
class MyDataset(Dataset):
def __init__(self,dataset):
self.dataset = dataset
def __getitem__(self,index):
data,target = self.dataset[index]
return data,target,index
def __len__(self):
return len(self.dataset )<prepare_x_and_y>
COLOR = 'black'
mpl.rcParams['text.color'] = COLOR
mpl.rcParams['axes.labelcolor'] = COLOR
mpl.rcParams['xtick.color'] = COLOR
mpl.rcParams['ytick.color'] = COLOR
plt.rcParams.update({'font.size': 18})
 plt.subplots_adjust(wspace = 15, hspace = 15 )
def sigmoid(x):
return 1 /(1 + np.exp(-x))
if debug :
x_test_cuda = torch.tensor(x_test, dtype=torch.long)
 else :
x_test_cuda = torch.tensor(x_test, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False )<split>
original_training_df = pd.read_csv('/kaggle/input/train.csv')
 original_training_df.head()
def kfold_train(x_train,y_train,x_test, model_obj, train_features, test_features,clip = True):
avg_losses_f = []
avg_val_losses_f = []
train_preds = np.zeros(( len(x_train)))
 test_preds = np.zeros(( len(x_test)))
 splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True ).split(x_train, y_train))
for i,(train_idx, valid_idx)in tqdm(enumerate(splits)) :
x_train = np.array(x_train)
 y_train = np.array(y_train)
 x_train_fold = torch.tensor(x_train[train_idx.astype(int)], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(y_train[train_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(x_train[valid_idx.astype(int)], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(y_train[valid_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
feature_train_fold = train_features[train_idx.astype(int)]
feature_valid_fold = train_features[valid_idx.astype(int)]
model = copy.deepcopy(model_obj)
 model.cuda()
train_ds = MyDataset(torch.utils.data.TensorDataset(x_train_fold, y_train_fold))
valid_ds = MyDataset(torch.utils.data.TensorDataset(x_val_fold, y_val_fold))
train_dl = torch.utils.data.DataLoader(train_ds, batch_size=batch_size, shuffle=True)
 valid_dl = torch.utils.data.DataLoader(valid_ds, batch_size=batch_size, shuffle=False)
 optimizer = torch.optim.Adam(model.parameters())
 scheduler = optim.lr_scheduler.ExponentialLR(optimizer,0.01)
 print('Fold %d'%i)
 for epoch in tqdm(range(n_epochs)) :
start_time = time.time()
avg_loss = 0.
 model.train()
for i,(x_batch, y_batch, index)in enumerate(train_dl):
y_pred = model(x_batch,torch.FloatTensor(feature_train_fold[index] ).cuda())
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
if clip:
nn.utils.clip_grad_norm_(model.parameters() ,1)
 optimizer.step()
avg_loss += loss.item() / len(train_dl)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(( len(x_test)))
 avg_val_loss = 0.
 for i,(x_batch, y_batch,index)in enumerate(valid_dl):
y_pred = model(x_batch,torch.FloatTensor(feature_valid_fold[index] ).cuda() ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_dl)
 valid_preds_fold[index] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s' 
.format(epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))
avg_losses_f.append(avg_loss)
 avg_val_losses_f.append(avg_val_loss)
 for i,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch,torch.FloatTensor(test_features[i * batch_size:(i+1)* batch_size] ).cuda())
 test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.detach().cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits)
 print('All \t loss={:.4f} \t val_loss={:.4f} \t '.format(np.average(avg_losses_f),np.average(avg_val_losses_f)))
 return train_preds, test_preds<split>
original_training_df.isnull().sum()
def cpu_kfold_train(x_train,y_train,x_test, model_obj, train_features, test_features,clip = True):
avg_losses_f = []
avg_val_losses_f = []
train_preds = np.zeros(( len(x_train)))
 test_preds = np.zeros(( len(x_test)))
 splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True ).split(x_train, y_train))
for i,(train_idx, valid_idx)in tqdm(enumerate(splits)) :
x_train = np.array(x_train)
 y_train = np.array(y_train)
 x_train_fold = torch.tensor(x_train[train_idx.astype(int)], dtype=torch.long)
 y_train_fold = torch.tensor(y_train[train_idx.astype(int), np.newaxis], dtype=torch.float32)
 x_val_fold = torch.tensor(x_train[valid_idx.astype(int)], dtype=torch.long)
 y_val_fold = torch.tensor(y_train[valid_idx.astype(int), np.newaxis], dtype=torch.float32)
 feature_train_fold = train_features[train_idx.astype(int)]
feature_valid_fold = train_features[valid_idx.astype(int)]
model = copy.deepcopy(model_obj)
 train_ds = MyDataset(torch.utils.data.TensorDataset(x_train_fold, y_train_fold))
valid_ds = MyDataset(torch.utils.data.TensorDataset(x_val_fold, y_val_fold))
train_dl = torch.utils.data.DataLoader(train_ds, batch_size=batch_size, shuffle=True)
 valid_dl = torch.utils.data.DataLoader(valid_ds, batch_size=batch_size, shuffle=False)
 optimizer = torch.optim.Adam(model.parameters())
 scheduler = optim.lr_scheduler.ExponentialLR(optimizer,0.01)
 print(f'Fold {i + 1}')
 for epoch in tqdm(range(n_epochs)) :
start_time = time.time()
avg_loss = 0.
 model.train()
for i,(x_batch, y_batch, index)in tqdm(enumerate(train_dl)) :
y_pred = model(x_batch,torch.FloatTensor(feature_train_fold[index]))
loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
if clip:
nn.utils.clip_grad_norm_(model.parameters() ,1)
 optimizer.step()
avg_loss += loss.item() / len(train_dl)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(( len(x_test)))
 avg_val_loss = 0.
 for i,(x_batch, y_batch,index)in tqdm(enumerate(valid_dl)) :
y_pred = model(x_batch,torch.FloatTensor(feature_valid_fold[index])).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_dl)
 valid_preds_fold[index] = sigmoid(y_pred.detach().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))
avg_losses_f.append(avg_loss)
 avg_val_losses_f.append(avg_val_loss)
 for i,(x_batch,)in tqdm(enumerate(test_loader)) :
y_pred = model(x_batch,torch.FloatTensor(test_features[i * batch_size:(i+1)* batch_size]))
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.detach().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits)
 print('All \t loss={:.4f} \t val_loss={:.4f} \t '.format(np.average(avg_losses_f),np.average(avg_val_losses_f)))
 return train_preds, test_preds<choose_model_class>
original_training_df.isnull().sum().divide(len(original_training_df.index)).multiply(100 )
model = NeuralNet(embedding_matrix,maxlen=maxlen,max_features=len(embedding_matrix))<train_model>
def get_title(dataframe_in):
dataframe_in['Title'] = dataframe_in['Name'].apply(lambda X: re.search('[A-Z]{1}[a-z]+\.', X ).group(0))
return dataframe_in
dataframe_transformations_test = original_training_df.copy()
dataframe_transformations_test = get_title(dataframe_transformations_test)
 dataframe_transformations_test.head()
if debug:
train_preds, test_preds = cpu_kfold_train(x_train,y_train,x_test,model,train_features,test_features)
 else :
train_preds, test_preds = kfold_train(x_train,y_train,x_test,model,train_features,test_features )<compute_train_metric>
dataframe_transformations_test.drop('PassengerId', axis = 1, inplace = True )
def findthreshold(y_train, train_preds):
best_f1= 0
best_thresh= 0
for thresh in tqdm(np.arange(0.1,0.501,0.01)) :
f1 = f1_score(y_train,(train_preds>thresh))
if f1 > best_f1:
best_thresh = thresh
best_f1 = f1
print('best threshold is {:.4f} with F1 score: {:.4f}'.format(best_thresh, best_f1))
return best_thresh, best_f1
threshold, score = findthreshold(y_train,train_preds )<save_to_csv>
def convert_sex_to_number(dataframe_in):
dataframe_in['Sex'] = dataframe_in['Sex'].apply(lambda X: 0 if X == 'female' else 1)
 return dataframe_in
dataframe_transformations_test = convert_sex_to_number(dataframe_transformations_test )
if debug:
df_test = pd.read_csv(".. /input/quora-insincere-questions-classification/test.csv")[:200]
else:
df_test = pd.read_csv(".. /input/quora-insincere-questions-classification/test.csv")
 submission = df_test[['qid']].copy()
submission['prediction'] =(test_preds > threshold ).astype(int)
 submission.to_csv('submission.csv', index=False )<set_options>
def get_cabin_letter(dataframe_in):
dataframe_in['Cabin'] = dataframe_in['Cabin'].apply(lambda X: re.search('[A-Za-z]{1}', X ).group(0 ).upper() if isinstance(X, str)else '?')
 return dataframe_in
def get_ticket_prefix(dataframe_in):
dataframe_in['Ticket'] = dataframe_in['Ticket'].apply(lambda X: X.split(' ')[0] if len(X.split(' ')) > 1 else '?')
 return dataframe_in
dataframe_transformations_test = get_cabin_letter(dataframe_transformations_test)
 dataframe_transformations_test = get_ticket_prefix(dataframe_transformations_test)
 dataframe_transformations_test.head()
tqdm.pandas()<init_hyperparams>
def get_family_name(dataframe_in):
dataframe_in['FamilyName'] = dataframe_in['Name'].apply(lambda X: re.search('[A-Z]{1}[a-z ]+', X ).group(0))
return dataframe_in
dataframe_transformations_test = get_family_name(dataframe_transformations_test)
 dataframe_transformations_test.drop(['Name'], axis = 1, inplace = True)
 dict_cabins_per_family = dict()
for current_family_name in dataframe_transformations_test.FamilyName.unique().tolist() :
filter_family_name =(dataframe_transformations_test['FamilyName'] == current_family_name)
 filter_known_cabin =(dataframe_transformations_test['Cabin'] != '?')
 listCabinsFromFamily = dataframe_transformations_test.loc[(filter_family_name)&\ 
(filter_known_cabin)].Cabin.unique().tolist()
if len(listCabinsFromFamily)> 0:
max_v, mode = 0, None
for curr_cabin, v in Counter(listCabinsFromFamily ).items() :
if v > max_v:
max_v, mode = v, curr_cabin
dict_cabins_per_family[current_family_name] = mode
def get_family_cabin_per_row(df_row):
if df_row.FamilyName in dict_cabins_per_family and df_row.Cabin == '?':
out = dict_cabins_per_family[df_row.FamilyName]
else:
out = df_row.Cabin
return out
def get_family_cabin(df):
df['Cabin'] = df.apply(get_family_cabin_per_row, axis = 1)
 df['Cabin'] = df['Cabin'].fillna('?')
 return df
dataframe_transformations_test['Cabin'] = get_family_cabin(dataframe_transformations_test)
 dataframe_transformations_test.isnull().sum()
emb_size = 300
max_features = 200000
maxlen = 100<set_options>
def embarked_fillna_median(df):
df.loc[:, ['Embarked']] = df['Embarked'].fillna(value = df['Embarked'].value_counts().idxmax())
 return df
dataframe_transformations_test = embarked_fillna_median(dataframe_transformations_test)
 dataframe_transformations_test.isnull().sum()
def clean_memory(*args):
for arg in args:
del arg
gc.collect()
time.sleep(10 )<load_from_csv>
def fill_age_from_masters(df, strategy_in = 'median'):
is_master =(df['Title'] == 'Master.')
 imp = SimpleImputer(missing_values = np.nan, strategy = strategy_in)
 df.loc[is_master, 'Age'] = imp.fit_transform(df.loc[is_master][['Age']])
 return df
dataframe_transformations_test = fill_age_from_masters(dataframe_transformations_test)
 dataframe_transformations_test.isnull().sum()
path = '/kaggle/input/quora-insincere-questions-classification'
train_df = pd.read_csv(path + "/train.csv")
 test_df = pd.read_csv(path + "/test.csv")
 train_df["question_text"].fillna("_na_", inplace=True)
 test_df["question_text"].fillna("_na_", inplace=True )<feature_engineering>
def fill_age_from_non_masters(df, strategy_in = 'median'):
is_not_master =(df['Title'] != 'Master.')
 imp = SimpleImputer(missing_values = np.nan, strategy = strategy_in)
 df.loc[is_not_master, 'Age'] = imp.fit_transform(df.loc[is_not_master][['Age']])
 return df
dataframe_transformations_test = fill_age_from_non_masters(dataframe_transformations_test)
 dataframe_transformations_test.isnull().sum()
train_df["num_words"] = train_df["question_text"].progress_apply(lambda x: len(str(x ).split()))
train_df["num_unique_words"] = train_df["question_text"].progress_apply(lambda x: len(set(str(x ).split())))
 train_df["num_chars"] = train_df["question_text"].progress_apply(lambda x: len(str(x)))
<feature_engineering>
def get_family_info(df_in):
df_in['FamilyMembers'] = df_in['Parch'] + df_in['SibSp'] + 1
df_in['Is_Mother'] = np.where(( df_in.Title=='Mrs.')&(df_in.Parch >0), 1, 0)
 return df_in
test_df["num_words"] = test_df["question_text"].progress_apply(lambda x: len(str(x ).split()))
test_df["num_unique_words"] = test_df["question_text"].progress_apply(lambda x: len(set(str(x ).split())))
 test_df["num_chars"] = test_df["question_text"].progress_apply(lambda x: len(str(x)) )<feature_engineering>
class TransformerSignificantData(BaseEstimator, TransformerMixin):
def __init__(self):
return
def fit(self, X, y = None):
return self
def transform(self, X, y = None):
out = get_cabin_letter(X)
 out = get_ticket_prefix(out)
 out = get_title(out)
 out = get_family_info(out)
 out = classify_title(out)
 out = get_family_name(out)
 out = get_family_cabin(out)
 return out
def build_vocab(sentences, verbose = True):
vocab = {}
for sentence in tqdm(sentences, disable =(not verbose)) :
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab<feature_engineering>
class TransformerDummify(BaseEstimator, TransformerMixin):
def __init__(self):
return
def fit(self, X, y = None):
return self
def transform(self, X, y = None):
columns_to_dummify = ['Sex', 'Cabin', 'Embarked', 'Title', 'Ticket']
useless_columns = ['PassengerId', 'Name', 'FamilyName']
dim_redundant_cols = ['Sex_male', 'Cabin_?', 'Embarked_S', 'Title_Rare', 'Ticket_?']
out_dummies = pd.get_dummies(X[columns_to_dummify], prefix = columns_to_dummify)
 out = pd.concat([X, out_dummies], axis = 1)
 out = out.drop(useless_columns + columns_to_dummify + dim_redundant_cols, axis = 1)
 return out
emb_path = '/kaggle/input/quora-insincere-questions-classification/embeddings'
def load_all() :
word2vec_format = {}
glove = [o.split(" ")[0] for o in tqdm(open(emb_path + '/glove.840B.300d/glove.840B.300d.txt')) ]
for word in tqdm(glove):
word2vec_format[word] = 1
clean_memory(glove)
 return word2vec_format
emb_all = load_all()<categorify>
class TransformerMissingData(BaseEstimator, TransformerMixin):
def __init__(self, missing_age_masters_strategy = 'mean', missing_age_non_masters_strategy = 'mean'):
self.missing_age_masters_strategy = missing_age_masters_strategy
self.missing_age_non_masters_strategy = missing_age_non_masters_strategy
def fit(self, X, y = None):
return self
def transform(self, X, y = None):
is_not_master =(X['Title_Master.'] == 0)
 is_master =(X['Title_Master.'] == 1)
 imp_non_master = SimpleImputer(missing_values = np.nan,
strategy = self.missing_age_non_masters_strategy)
 imp_master = SimpleImputer(missing_values = np.nan,
strategy = self.missing_age_masters_strategy)
 imp_fare = SimpleImputer(missing_values = np.nan,
strategy = 'mean')
 X.loc[is_not_master, 'Age'] = imp_non_master.fit_transform(X.loc[is_not_master, ['Age']])
 X.loc[is_master, 'Age'] = imp_master.fit_transform(X.loc[is_master, ['Age']])
 X.loc[:, 'Fare'] = imp_fare.fit_transform(X.loc[:, ['Fare']])
 return X
def check_coverage(vocab, embeddings_index):
a = {}
oov = {}
k = 0
i = 0
for word in tqdm(vocab):
try:
a[word] = embeddings_index[word]
k += vocab[word]
except:
oov[word] = vocab[word]
i += vocab[word]
pass
print('Found embeddings for {:.2%} of vocab'.format(len(a)/ len(vocab)))
 print('Found embeddings for {:.2%} of all text'.format(k /(k + i)))
 sorted_x = sorted(oov.items() , key=operator.itemgetter(1)) [::-1]
return sorted_x<split>
class TransformerNormalize(BaseEstimator, TransformerMixin):
def __init__(self):
return
def fit(self, X, y = None):
return self
def transform(self, X, y = None):
out = X
out.loc[:, ['Pclass']] = MinMaxScaler().fit_transform(out[['Pclass']])
 out.loc[:, ['Age', 'Fare']] = StandardScaler().fit_transform(out[['Age', 'Fare']])
 return out
sentences = train_df["question_text"].progress_apply(lambda x: x.split())
 vocab = build_vocab(sentences)
 oov = check_coverage(vocab, emb_all)
 oov[:10]<define_variables>
df_dummy = pd.concat([pd.read_csv('/kaggle/input/train.csv'), pd.read_csv('/kaggle/input/test.csv')], ignore_index = True)
 dummy_pipeline = Pipeline([ 
('prepare', TransformerSignificantData()), 
('dummify', TransformerDummify()), 
('normalize', TransformerNormalize())
 ])
 df_dummy = dummy_pipeline.transform(df_dummy)
 df_dummy.head()
punct = "/-'?!.,
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }
def clean_special_chars(text, punct=punct, mapping=punct_mapping):
for p in mapping:
if p in text:
text = text.replace(p, mapping[p])
 for p in punct:
if p in text:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
if s in text:
text = text.replace(s, specials[s])
 return text<feature_engineering>
related_to_age = ['Pclass', 'Title_Miss.', 'SibSp', 'Cabin_C', 'Title_Mr.', 'Fare', 'Parch']
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_special_chars(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: clean_special_chars(x))
sentences = train_df["question_text"].progress_apply(lambda x: x.split())
 vocab = build_vocab(sentences)
 oov = check_coverage(vocab,emb_all)
 oov[:10]<define_variables>
df_pca = df_dummy.copy().loc[:, related_to_age]
df_pca.isnull().sum()
mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization', 'pokémon': 'pokemon'}
def correct_spelling(x, dic):
for word in dic.keys() :
if word in x:
x = x.replace(word, dic[word])
 return x<feature_engineering>
df_pca = df_pca.fillna(value = df_pca['Fare'].mean())
 df_pca.isnull().sum()
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: correct_spelling(x, mispell_dict))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: correct_spelling(x, mispell_dict))
sentences = train_df["question_text"].apply(lambda x: x.split())
 vocab = build_vocab(sentences)
 oov = check_coverage(vocab, emb_all)
 oov[:10]<feature_engineering>
np.cumsum(pca_age.explained_variance_ratio_ )
go_to_more_common_words = {
'Redmi': 'Mobile',
'OnePlus': 'Mobile',
'Quorans': 'Quoran',
'cryptocurrencies': 'technology',
'Cryptocurrency': 'Technology',
'Blockchain': 'Technology',
'Upwork': 'Technology',
'HackerRank': 'Programming',
}
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: correct_spelling(x, go_to_more_common_words))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: correct_spelling(x, go_to_more_common_words))
sentences = train_df["question_text"].apply(lambda x: x.split())
 vocab = build_vocab(sentences)
 oov = check_coverage(vocab, emb_all)
 oov[:10]<drop_column>
kmeans2 = KMeans(n_clusters = 2, random_state = 42, n_init = 500 ).fit_predict(np_pca_significant)
 kmeans3 = KMeans(n_clusters = 3, random_state = 42, n_init = 500 ).fit_predict(np_pca_significant)
 kmeans4 = KMeans(n_clusters = 4, random_state = 42, n_init = 500 ).fit_predict(np_pca_significant)
 kmeans5 = KMeans(n_clusters = 5, random_state = 42, n_init = 500 ).fit_predict(np_pca_significant)
 kmeans6 = KMeans(n_clusters = 6, random_state = 42, n_init = 500 ).fit_predict(np_pca_significant )
clean_memory(oov, vocab, sentences, mispell_dict, go_to_more_common_words, punct, punct_mapping, emb_all )<split>
dbscan_model = DBSCAN(eps = 0.7, min_samples = 20 ).fit(np_pca_significant)
 dbscan_predict = dbscan_model.fit_predict(np_pca_significant )
def load_all_emb(word_index):
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 glove = dict(get_coefs(*o.split(" ")) for o in tqdm(open(emb_path + '/glove.840B.300d/glove.840B.300d.txt')) if o.split(" ")[0] in word_index)
 global max_features
all_embs = np.stack(glove.values())
 emb_mean, emb_std = all_embs.mean() , all_embs.std()
clean_memory(all_embs)
 nb_words = min(max_features, len(word_index))
print(nb_words)
 max_features = nb_words
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, emb_size))
for word, i in tqdm(word_index.items()):
if i >= nb_words: continue
glove_vector = glove.get(word)
 paragram_vector = None
if glove_vector is not None and paragram_vector is not None:
embedding_matrix[i] = 0.7 * glove_vector + 0.3 * paragram_vector
elif glove_vector is not None:
embedding_matrix[i] = glove_vector
elif paragram_vector is not None:
embedding_matrix[i] = paragram_vector
clean_memory(glove_vector, paragram_vector, glove)
 return embedding_matrix<load_pretrained>
class TransformerMissingData(BaseEstimator, TransformerMixin):
def __init__(self, missing_age_masters_strategy = 'mean', missing_age_non_masters_strategy = 'cluster'):
self.missing_age_masters_strategy = missing_age_masters_strategy
self.missing_age_non_masters_strategy = missing_age_non_masters_strategy
def fit(self, X, y = None):
return self
def transform(self, X, y = None):
is_not_master =(X['Title_Master.'] == 0)
 is_master =(X['Title_Master.'] == 1)
 imp_non_master = SimpleImputer(missing_values = np.nan,
strategy = self.missing_age_non_masters_strategy)
 imp_master = SimpleImputer(missing_values = np.nan,
strategy = self.missing_age_masters_strategy)
 imp_fare = SimpleImputer(missing_values = np.nan,
strategy = 'mean')
 X.loc[is_master, 'Age'] = imp_master.fit_transform(X.loc[is_master, ['Age']])
 X.loc[:, 'Fare'] = imp_fare.fit_transform(X.loc[:, ['Fare']])
 if self.missing_age_non_masters_strategy in ['mean', 'median']:
X.loc[is_not_master, 'Age'] = imp_non_master.fit_transform(X.loc[is_not_master, ['Age']])
 elif self.missing_age_non_masters_strategy == 'cluster':
X_without_age = X.drop(['Age', 'Survived'], axis = 1)
 print(X_without_age.isnull().sum())
 pca_fit_vec = PCA(n_components = 3 ).fit_transform(X_without_age)
 X['ClusterLabel'] = DBSCAN(eps = 0.7, min_samples = 20 ).fit_predict(pca_fit_vec)
 dict_means_per_cluster = dict()
for cluster_label in X['ClusterLabel'].tolist() :
dict_means_per_cluster[cluster_label] = X.loc[X['ClusterLabel'] == cluster_label, :]['Age'].mean()
age_list = X['Age'].tolist()
cluster_list = X['ClusterLabel'].tolist()
for i, curr_age in enumerate(age_list):
if np.isnan(curr_age):
age_list[i] = dict_means_per_cluster[cluster_label]
X['Age'] = age_list
return X
emb_path = '/kaggle/input/quora-insincere-questions-classification/embeddings'
tokenizer = Tokenizer(num_words=max_features, filters="")
 tokenizer.fit_on_texts(np.concatenate(( train_df["question_text"].values, test_df["question_text"].values)))
 embeddings = load_all_emb(tokenizer.word_index)
 print(max_features )<split>
feature_engineering_pipeline = Pipeline([ 
('prepare', TransformerSignificantData()), 
('dummify', TransformerDummify()), 
('normalize', TransformerNormalize()), 
('missing', TransformerMissingData())
 ])
 df_all_original = pd.concat([pd.read_csv('/kaggle/input/train.csv'),
pd.read_csv('/kaggle/input/test.csv')], ignore_index = True)
 df_all_original.head()
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=47)
 train_X = train_df["question_text"].values
val_X = val_df["question_text"].values
test_X = test_df["question_text"].values
train_X2 = train_df[['num_words', 'num_unique_words', 'num_chars']]
val_X2 = val_df[['num_words', 'num_unique_words', 'num_chars']]
test_X2 = test_df[['num_words', 'num_unique_words', 'num_chars']]
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
print('Train shape: {}'.format(train_X.shape))
print('Validation shape: {}'.format(val_X.shape))
print('Test shape: {}'.format(test_X.shape))
clean_memory(train_df, val_df, test_df )<choose_model_class>
df_all_processed = feature_engineering_pipeline.transform(df_all_original.copy())
 df_all_processed.head()
early_stopping = EarlyStopping(monitor="val_f1", patience=2, restore_best_weights=True, mode="max")
 reduce_lr = ReduceLROnPlateau(monitor="val_f1", mode="max")
 callbacks = [early_stopping, reduce_lr]<compute_test_metric>
df_all_processed.isnull().sum()
def f1(y_true, y_pred):

 def recall(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))<choose_model_class>
df_ans = df_all_processed[df_all_processed['Survived'].isnull() ]
df_ans = df_ans.loc[:, df_ans.columns != 'Survived']
df_cv_and_test = df_all_processed[df_all_processed['Survived'].notnull() ]
df_cv_and_test_X = df_cv_and_test.drop(['Survived', 'ClusterLabel'], axis = 1)
 df_cv_and_test_Y = df_cv_and_test['Survived']
df_cv_X, df_test_X, df_cv_Y, df_test_Y = train_test_split(df_cv_and_test_X,
df_cv_and_test_Y,
test_size = 0.1,
random_state = 42 )
inp1 = Input(shape=(maxlen,))
x = Embedding(max_features, emb_size, weights=[embeddings], trainable=False )(inp1)
 x = SpatialDropout1D(0.4 )(x)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Conv1D(64, 1 )(x)
 max_pool = GlobalMaxPooling1D()(x)
 inp2 = Input(shape=(3,))
y = Dense(64 )(inp2)
 xy = concatenate([max_pool, y], axis=1)
 xy = Dense(128, activation="relu" )(xy)
 xy = Dropout(0.1 )(xy)
 xy = BatchNormalization()(xy)
 outp = Dense(1, activation="sigmoid" )(xy)
 model = Model(inputs=[inp1, inp2], outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[f1] )<train_model>
hyper_rf = dict(
 n_estimators = [85, 86, 87, 88, 89],
max_depth = [3, 4, 5, 6, 7, 8, 9, 10],
random_state = [42] 
)
 hyper_svm = dict(
 C = [6.0, 7.0, 8.0],
kernel = ['linear', 'poly', 'rbf', 'sigmoid'],
gamma = ['auto', 'scale'],
probability = [True],
random_state = [42] 
)
 hyper_logit = dict(
 C = [0.1, 1.0, 10.0],
solver = ['lbfgs'],
random_state = [42] 
)
 hyper_adaboost = dict(
 n_estimators = [54, 55, 56],
learning_rate = [0.35, 0.4, 0.45],
random_state = [42] 
)
 hyper_xgboost = {
'eta': [0, 0.00001],
'gamma': [0.001, 0.05, 0.1],
'max_depth': [3, 4, 5],
'probability': [True],
'random_state': [42]
}
model.fit([train_X, train_X2], train_y, batch_size=512, epochs=20, validation_data=([val_X, val_X2], val_y), callbacks=callbacks, verbose=True )<drop_column>
n_cv = 10
gscv_rf = GridSearchCV(RandomForestClassifier() , hyper_rf, scoring = 'accuracy', cv = n_cv)
 gscv_svm = GridSearchCV(SVC() , hyper_svm, scoring = 'accuracy', cv = n_cv)
 gscv_logit = GridSearchCV(LogisticRegression() , hyper_logit, scoring = 'accuracy', cv = n_cv)
 gscv_adab = GridSearchCV(AdaBoostClassifier() , hyper_adaboost, scoring = 'accuracy', cv = n_cv)
 gscv_xbt = GridSearchCV(XGBClassifier() , hyper_xgboost, scoring = 'accuracy', cv = n_cv )
clean_memory(train_X, train_y )<find_best_model_class>
print('1.Fitting Random Forest')
 gscv_rf.fit(df_cv_X.copy() , df_cv_Y.copy())
 print('2.Fitting SVM')
 gscv_svm.fit(df_cv_X.copy() , df_cv_Y.copy())
 print('3.Fitting Logit')
 gscv_logit.fit(df_cv_X.copy() , df_cv_Y.copy())
 print('4.Fitting AdaBoost')
 gscv_adab.fit(df_cv_X.copy() , df_cv_Y.copy())
 print('5.Fitting XBoostTree')
 gscv_xbt.fit(df_cv_X.copy() , df_cv_Y.copy() )
pred_val_y = model.predict([val_X, val_X2], batch_size=512, verbose=1)
 def scoring(y_true, y_proba, verbose=True):
def threshold_search(y_true, y_proba):
precision , recall, thresholds = precision_recall_curve(y_true, y_proba)
 thresholds = np.append(thresholds, 1.001)
 F = 2 /(1/precision + 1/recall)
 best_score = np.max(F)
 best_th = thresholds[np.argmax(F)]
return best_th
rkf = RepeatedStratifiedKFold(n_splits=5, n_repeats=10)
 scores = []
ths = []
for train_index, test_index in rkf.split(y_true, y_true):
y_prob_train, y_prob_test = y_proba[train_index], y_proba[test_index]
y_true_train, y_true_test = y_true[train_index], y_true[test_index]
best_threshold = threshold_search(y_true_train, y_prob_train)
 sc = f1_score(y_true_test,(y_prob_test >= best_threshold ).astype(int))
scores.append(sc)
 ths.append(best_threshold)
 best_th = np.mean(ths)
 score = np.mean(scores)
 if verbose: print(f'Best threshold: {np.round(best_th, 4)}, Score: {np.round(score,5)}')
 return best_th, score
optimal_point1, optimal1 = scoring(val_y, pred_val_y)
 print("Optimal F1 {} on threshhold {}".format(optimal1, optimal_point1))<compute_test_metric>
def get_CV_score(classifier_name, n_cv):
ans = [0] * n_cv
for i in range(0, n_cv):
command_str = classifier_name + '.cv_results_['split' + str(i)+ '_test_score']'
ans[i] = eval(command_str)[0]
return ans
train_score_list_rf = get_CV_score('gscv_rf', n_cv)
 train_score_list_svm = get_CV_score('gscv_svm', n_cv)
 train_score_list_logit = get_CV_score('gscv_logit', n_cv)
 train_score_list_adab = get_CV_score('gscv_adab', n_cv)
 train_score_list_xbt = get_CV_score('gscv_xbt', n_cv)
 df_train_score = pd.DataFrame(dict(
 estimator = ['1.rf'] * n_cv + ['2.svm'] * n_cv + ['3.logit'] * n_cv + ['4.adaboost'] * n_cv + ['5.xboosttree'] * n_cv,
score = train_score_list_rf + train_score_list_svm + train_score_list_logit + train_score_list_adab + train_score_list_xbt 
))
df_test_score = pd.DataFrame(dict(
 estimator = ['1.rf'] + ['2.svm'] + ['3.logit'] + ['4.adaboost'] + ['5.xboosttree'],
score = [accuracy_score(gscv_rf.best_estimator_.predict(df_test_X), df_test_Y),
accuracy_score(gscv_svm.best_estimator_.predict(df_test_X), df_test_Y),
accuracy_score(gscv_logit.best_estimator_.predict(df_test_X), df_test_Y),
accuracy_score(gscv_adab.best_estimator_.predict(df_test_X), df_test_Y),
accuracy_score(gscv_xbt.best_estimator_.predict(df_test_X), df_test_Y)] 
))
ggplot(aes(x = 'estimator', color = 'estimator', y = 'score')) +\
geom_boxplot(data = df_train_score)+\
geom_jitter(data = df_train_score)+ coord_cartesian(ylim = [0.7, 1])+\
geom_bar(data = df_test_score, stat = 'identity', alpha = 0.2, position = "dodge" )
clean_memory(val_X, val_y, pred_val_y )<predict_on_test>
def print_opt_hyper(dict_hyper, model):
for k in dict_hyper.keys() :
print(k + ': ' + str(model.best_estimator_.get_params() [k])+ ' -- ' + str(dict_hyper[k]))
print('Optimal Hyperparameters:')
 print('---')
 print('1.Random Forest')
 print_opt_hyper(hyper_rf, gscv_rf)
 print('---')
 print('2.SVM')
 print_opt_hyper(hyper_svm, gscv_svm)
 print('---')
 print('3.Logistic Regression')
 print_opt_hyper(hyper_logit, gscv_logit)
 print('---')
 print('4.Ada Boost')
 print_opt_hyper(hyper_adaboost, gscv_adab)
 print('---')
 print('5.XGBTree')
 print_opt_hyper(hyper_xgboost, gscv_xbt)
 print('---' )
all_preds = model.predict([test_X, test_X2], batch_size=512, verbose=1)
 pred_test_y =(np.array(all_preds)> optimal_point1 ).astype(np.int )<save_to_csv>
estimator_list = [('rf', gscv_rf.best_estimator_), 
('svm', gscv_svm.best_estimator_), 
('logit', gscv_logit.best_estimator_), 
('adab', gscv_adab.best_estimator_), 
('xbt', gscv_xbt.best_estimator_)]
hard_vote_estimator = VotingClassifier(estimator_list)
 soft_vote_estimator = VotingClassifier(estimator_list)
 meta_logit_estimator = StackingClassifier(classifiers = [X[1] for X in estimator_list],
meta_classifier = LogisticRegression() )
submit_df = pd.DataFrame({"qid": test_df["qid"], "prediction": pred_test_y1})
 submit_df.to_csv("submission.csv", index=False )<import_modules>
%%capture
hard_cv = GridSearchCV(hard_vote_estimator, param_grid = {'voting': ['hard']},
scoring = 'accuracy', cv = n_cv ).fit(df_cv_X.copy() , df_cv_Y.copy())
 soft_cv = GridSearchCV(soft_vote_estimator, param_grid = {'voting': ['soft']},
scoring = 'accuracy', cv = n_cv ).fit(df_cv_X.copy() , df_cv_Y.copy())
 stack_cv = GridSearchCV(meta_logit_estimator, param_grid = {'meta_classifier__C': [0.1, 1, 10, 100, 500]},
scoring = 'accuracy', cv = n_cv ).fit(df_cv_X.copy() , df_cv_Y.copy() )
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.initializers import he_normal, he_uniform, glorot_normal, glorot_uniform
from keras import backend as K
from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau
from keras.models import Model
from keras.layers import Dense, Embedding, Bidirectional, CuDNNGRU, GlobalAveragePooling1D
from keras.layers import CuDNNLSTM, GlobalMaxPooling1D, concatenate, Input, Dropout, SpatialDropout1D
from keras.optimizers import Adam
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.metrics import f1_score
import matplotlib.pyplot as plt
from tqdm import tqdm
from statistics import mean, median, stdev
from numpy import amax, amin
import gc
import time
import pandas as pd
import numpy as np
import tensorflow as tf
import operator
import re
import os
import math<define_variables>
print('Hard Voting Score: ' + str(hard_cv.best_score_))
print('Soft Voting Score: ' + str(soft_cv.best_score_))
print('Stacking Score: ' + str(stack_cv.best_score_))
batch_size = 1024
epochs = 18
current_embd = "Glove"
question_length = 100
max_eval_size = 15000
DATA_SPLIT_SEED = 2018
K_FOLDS = 5
K_FOLD_EPOCHS = int(epochs/K_FOLDS)
 seed_nb=14
np.random.seed(seed_nb)
 tf.set_random_seed(seed_nb )<define_variables>
train_score_list_hv = get_CV_score('hard_cv', n_cv)
 train_score_list_sv = get_CV_score('soft_cv', n_cv)
 train_score_list_st = get_CV_score('stack_cv', n_cv)
 df_train_score = pd.DataFrame(dict(
 estimator = ['1.rf'] * len(train_score_list_rf)+
['2.svm'] * len(train_score_list_svm)+
['3.logit'] * len(train_score_list_logit)+
['4.adaboost'] * len(train_score_list_adab)+
['5.xbtree'] * len(train_score_list_xbt)+
['6.hard voter'] * len(train_score_list_hv)+
['7.soft voter'] * len(train_score_list_sv)+
['8.stacking'] * len(train_score_list_st)
 , score = train_score_list_rf + train_score_list_svm + train_score_list_logit + train_score_list_adab +\
train_score_list_xbt + train_score_list_hv + train_score_list_sv + train_score_list_st 
))
df_test_score = pd.DataFrame(dict(
 estimator = ['1.rf'] + ['2.svm'] + ['3.logit'] + ['4.adaboost'] + ['5.xbtree'] + ['6.hard voter'] + ['7.soft voter'] + ['8.stacking'],
score = [accuracy_score(gscv_rf.best_estimator_.predict(df_test_X), df_test_Y),
accuracy_score(gscv_svm.best_estimator_.predict(df_test_X), df_test_Y),
accuracy_score(gscv_logit.best_estimator_.predict(df_test_X), df_test_Y),
accuracy_score(gscv_adab.best_estimator_.predict(df_test_X), df_test_Y),
accuracy_score(gscv_xbt.best_estimator_.predict(df_test_X), df_test_Y),
accuracy_score(hard_cv.best_estimator_.predict(df_test_X), df_test_Y),
accuracy_score(soft_cv.best_estimator_.predict(df_test_X), df_test_Y),
accuracy_score(stack_cv.best_estimator_.predict(df_test_X), df_test_Y),
] 
))
ggplot(aes(x = 'estimator', y = 'score', color = 'estimator')) + geom_boxplot(data = df_train_score)+\
geom_jitter(data = df_train_score)+ theme(axis_text_x = element_text(angle = 70,
hjust = 1, vjust = 0.5)) + coord_cartesian(ylim = [0.7, 1])+\
geom_bar(data = df_test_score, stat = 'identity', alpha = 0.2, position = "dodge" )
paraules_prohibides = ['2g1c', '2 girls 1 cup', 'acrotomophilia', 'alabama hot pocket', 'alaskan pipeline', 'anal', 'anilingus', 'anus',
'apeshit', 'arsehole', 'ass', 'asshole', 'assmunch', 'auto erotic', 'autoerotic', 'babeland', 'baby batter',
'baby juice', 'ball gag', 'ball gravy', 'ball kicking', 'ball licking', 'ball sack', 'ball sucking', 'bangbros',
'bareback', 'barely legal', 'barenaked', 'bastard', 'bastardo', 'bastinado', 'bbw', 'bdsm', 'beaner', 'beaners',
'beaver cleaver', 'beaver lips', 'bestiality', 'big black', 'big breasts', 'big knockers', 'big tits', 'bimbos',
'birdlock', 'bitch', 'bitches', 'black cock', 'blonde action', 'blonde on blonde action', 'blowjob', 'blow job',
'blow your load', 'blue waffle', 'blumpkin', 'bollocks', 'bondage', 'boner', 'boob', 'boobs', 'booty call',
'brown showers', 'brunette action', 'bukkake', 'bulldyke', 'bullet vibe', 'bullshit', 'bung hole', 'bunghole',
'busty', 'butt', 'buttcheeks', 'butthole', 'camel toe', 'camgirl', 'camslut', 'camwhore', 'carpet muncher',
'carpetmuncher', 'chocolate rosebuds', 'circlejerk', 'cleveland steamer', 'clit', 'clitoris', 'clover clamps',
'clusterfuck', 'cock', 'cocks', 'coprolagnia', 'coprophilia', 'cornhole', 'coon', 'coons', 'creampie', 'cum',
'cumming', 'cunnilingus', 'cunt', 'darkie', 'date rape', 'daterape', 'deep throat', 'deepthroat', 'dendrophilia',
'dick', 'dildo', 'dingleberry', 'dingleberries', 'dirty pillows', 'dirty sanchez', 'doggie style', 'doggiestyle',
'doggy style', 'doggystyle', 'dog style', 'dolcett', 'domination', 'dominatrix', 'dommes', 'donkey punch',
'double dong', 'double penetration', 'dp action', 'dry hump', 'dvda', 'eat my ass', 'ecchi', 'ejaculation',
'erotic', 'erotism', 'escort', 'eunuch', 'faggot', 'fecal', 'felch', 'fellatio', 'feltch', 'female squirting',
'femdom', 'figging', 'fingerbang', 'fingering', 'fisting', 'foot fetish', 'footjob', 'frotting', 'fuck',
'fuck buttons', 'fuckin', 'fucking', 'fucktards', 'fudge packer', 'fudgepacker', 'futanari', 'gang bang',
'gay sex', 'genitals', 'giant cock', 'girl on', 'girl on top', 'girls gone wild', 'goatcx', 'goatse', 'god damn',
'gokkun', 'golden shower', 'goodpoop', 'goo girl', 'goregasm', 'grope', 'group sex', 'g-spot', 'guro', 'hand job',
'handjob', 'hard core', 'hardcore', 'hentai', 'homoerotic', 'honkey', 'hooker', 'hot carl', 'hot chick', 'how to kill',
'how to murder', 'huge fat', 'humping', 'incest', 'intercourse', 'jack off', 'jail bait', 'jailbait', 'jelly donut',
'jerk off', 'jigaboo', 'jiggaboo', 'jiggerboo', 'jizz', 'juggs', 'kike', 'kinbaku', 'kinkster', 'kinky', 'knobbing',
'leather restraint', 'leather straight jacket', 'lemon party', 'lolita', 'lovemaking', 'make me come', 'male squirting',
'masturbate', 'menage a trois', 'milf', 'missionary position', 'motherfucker', 'mound of venus', 'mr hands', 'muff diver',
'muffdiving', 'nambla', 'nawashi', 'negro', 'neonazi', 'nigga', 'nigger', 'nig nog', 'nimphomania', 'nipple', 'nipples',
'nsfw images', 'nude', 'nudity', 'nympho', 'nymphomania', 'octopussy', 'omorashi', 'one cup two girls', 'one guy one jar',
'orgasm', 'orgy', 'paedophile', 'paki', 'panties', 'panty', 'pedobear', 'pedophile', 'pegging', 'penis', 'phone sex',
'piece of shit', 'pissing', 'piss pig', 'pisspig', 'playboy', 'pleasure chest', 'pole smoker', 'ponyplay', 'poof',
'poon', 'poontang', 'punany', 'poop chute', 'poopchute', 'porn', 'porno', 'pornography', 'prince albert piercing',
'pthc', 'pubes', 'pussy', 'queaf', 'queef', 'quim', 'raghead', 'raging boner', 'rape', 'raping', 'rapist', 'rectum',
'reverse cowgirl', 'rimjob', 'rimming', 'rosy palm', 'rosy palm and her 5 sisters', 'rusty trombone', 'sadism',
'santorum', 'scat', 'schlong', 'scissoring', 'semen', 'sexo', 'sexy', 'shaved beaver', 'shaved pussy',
'shemale', 'shibari', 'shit', 'shitblimp', 'shitty', 'shota', 'shrimping', 'skeet', 'slanteye', 'slut', 's&m',
'smut', 'snatch', 'snowballing', 'sodomize', 'sodomy', 'spic', 'splooge', 'splooge moose', 'spooge', 'spread legs',
'spunk', 'strap on', 'strapon', 'strappado', 'strip club', 'style doggy', 'suck', 'sucks', 'suicide girls', 'sultry women',
'swastika', 'swinger', 'tainted love', 'taste my', 'tea bagging', 'threesome', 'throating', 'tied up', 'tight white',
'tit', 'tits', 'titties', 'titty', 'tongue in a', 'topless', 'tosser', 'towelhead', 'tranny', 'tribadism', 'tub girl',
'tubgirl', 'tushy', 'twat', 'twink', 'twinkie', 'two girls one cup', 'undressing', 'upskirt', 'urethra play', 'urophilia',
'vagina', 'venus mound', 'vibrator', 'violet wand', 'vorarephilia', 'voyeur', 'vulva', 'wank', 'wetback', 'wet dream',
'white power', 'wrapping men', 'wrinkled starfish', 'xx', 'xxx', 'yaoi', 'yellow showers', 'yiffy', 'zoophilia', '🖕'] 
<load_from_csv>
model_ids = ['RandomForest', 'SVM', 'AdaBoost', 'Logistic', 'XBTree', 'SoftVoting', 'HardVoting', 'Stacking']
for i, model in enumerate([gscv_rf, gscv_svm, gscv_adab, gscv_logit, gscv_xbt, soft_cv, hard_cv, stack_cv]):
submission_estimator = model.best_estimator_.fit(df_cv_and_test_X, df_cv_and_test_Y)
 survival_ans_col = pd.DataFrame({'Survived': submission_estimator.predict(df_ans.drop('ClusterLabel', axis = 1, inplace = False)) })
 df_submission = pd.read_csv('/kaggle/input/test.csv')
 df_submission['Survived'] = [int(X)for X in survival_ans_col['Survived'].to_list() ]
df_submission = df_submission.loc[:, ['PassengerId', 'Survived']]
df_submission.to_csv('submission.csv')
 os.chdir('/kaggle/working')
 df_submission.to_csv('submission-' + model_ids[i] + '.csv', index = False)
 FileLinks('.' )