Kaggle competition: Titanic - Machine Learning from Disaster
predictions1 =(predictions>best_thresh ).astype(int )<save_to_csv>
a = pd.read_csv("/kaggle/input/titanic/train.csv")
 b = pd.read_csv("/kaggle/input/titanic/test.csv" )
predictions =(predictions>best_thresh ).astype(int)
 submission = pd.DataFrame({'qid': test_data.qid, 'prediction': predictions1[:,0]})
 submission.to_csv('submission.csv', index=False )<import_modules>
na = a.shape[0]
nb = b.shape[0]
frames= [a,b]
c1=pd.concat(frames, axis=0, sort=False ).reset_index(drop=True)
 target = a[['Survived']]
c1.drop(['Survived'], axis=1, inplace=True)
 print("The shape of the training set is", na)
 print("Total size is :",c1.shape )
import re
import time
import gc
import random
import os
import numpy as np
import pandas as pd
from tqdm import tqdm
from tabulate import tabulate
from sklearn.model_selection import train_test_split
from sklearn import metrics, preprocessing
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from sklearn.metrics import f1_score, roc_auc_score
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
import torch
import torch.nn as nn
import torch.utils.data<load_pretrained>
print(c1.isnull().sum() )
if not os.path.exists('./embeddings'):
os.mkdir('./embeddings')
 os.mkdir('./embeddings/glove.840B.300d/')
 os.mkdir('./embeddings/paragram_300_sl999/')
 with zipfile.ZipFile('.. /input/quora-insincere-questions-classification/embeddings.zip', 'r')as z:
with z.open('glove.840B.300d/glove.840B.300d.txt')as zf, open('./embeddings/glove.840B.300d/glove.840B.300d.txt', 'wb')as f:
shutil.copyfileobj(zf, f)
 with z.open('paragram_300_sl999/paragram_300_sl999.txt')as zf, open('./embeddings/paragram_300_sl999/paragram_300_sl999.txt', 'wb')as f:
shutil.copyfileobj(zf, f )<define_variables>
Survived=a['Survived'].value_counts()
Survived=pd.DataFrame(Survived)
 Survived=Survived.reset_index()
pclass= a['Pclass'].value_counts()
pclass= pd.DataFrame(pclass)
 pclass= pclass.reset_index()
sex= a['Sex'].value_counts()
sex= pd.DataFrame(sex)
 sex= sex.reset_index()
embarked=a['Embarked'].value_counts()
embarked= pd.DataFrame(embarked)
 embarked= embarked.reset_index()
sibling=a['SibSp'].value_counts()
sibling= pd.DataFrame(sibling)
 sibling= sibling.reset_index()
parch=a['Parch'].value_counts()
parch= pd.DataFrame(parch)
 parch= parch.reset_index()
plt.style.use('seaborn')
 sns.set_style('whitegrid')
 fig = plt.figure(figsize=(15,15))
ax1 = plt.subplot2grid(( 3,2),(0,0))
plt.pie(Survived.Survived,colors=("pink","r"), autopct='%2.1f%%',labels=Survived['index'], shadow=True)
 plt.title('Titanic survival rates', fontsize=13, weight='bold')
 ax1 = plt.subplot2grid(( 3,2),(0,1))
plt.pie(pclass.Pclass,colors=("cyan","lime", 'orange'), autopct='%2.1f%%',labels=pclass['index'], shadow=True)
 plt.title('Passengers class 1,2 and 3 distributions', fontsize=13, weight='bold')
 ax1 = plt.subplot2grid(( 3,2),(1,0))
plt.pie(sex.Sex,colors=("darkcyan","purple"), autopct='%2.1f%%',labels=sex['index'], shadow=True)
 plt.title('Passengers sex', fontsize=13, weight='bold')
 ax1 = plt.subplot2grid(( 3,2),(1,1))
plt.pie(embarked.Embarked,colors=("deepskyblue","lawngreen", 'crimson'), autopct='%2.1f%%',labels=embarked['index'], shadow=True)
 plt.title('Passengers embarking port', fontsize=13, weight='bold')
 ax1 = plt.subplot2grid(( 3,2),(2,0))
plt.pie(sibling.SibSp,colors=("gold","sienna","plum","deepskyblue","lawngreen", 'crimson'), autopct='%2.1f%%',labels=sibling['index'], shadow=True)
 plt.title('Passengers siblings number', fontsize=13, weight='bold')
 ax1 = plt.subplot2grid(( 3,2),(2,1))
plt.pie(parch.Parch,colors=("gold","sienna","plum","deepskyblue","lawngreen", 'crimson', 'red'), autopct='%2.1f%%',labels=parch['index'], shadow=True)
 plt.title('Passengers parents/children number', fontsize=13, weight='bold')
 plt.show()
embed_size = 300
meta_size = 3
max_features = 30000
maxlen = 72
batch_size = 1000
train_epochs = 4
SEED = 1029
def seed_torch(seed=1029):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True<define_variables>
c1['family_size']=c1['SibSp'] + c1['Parch'] + 1
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
''
def clean_and_lower_text(x):
x = str(x)
 x = x.encode("ascii", errors="ignore" ).decode()
for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
x = re.sub('[0-9]{1}', '
return x
mispell_dict = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not",
"didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not",
"he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will",
"how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am",
"I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am",
"i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have",
"it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not",
"mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not",
"needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not",
"sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will",
"she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have",
"so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is",
"there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would",
"they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have",
"to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have",
"we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are",
"what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is",
"where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is",
"why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not",
"wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are",
"y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are",
"you've": "you have", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling',
'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize',
'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are',
'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does',
'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist',
'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what',
'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 spaces_re = re.compile('\s')
 number_re = re.compile('[0-9]')
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
 def have_pattern(text, pattern_re):
return int(bool(pattern_re.search(text)) )<load_from_csv>
c1.isnull().sum()
def load_and_prec() :
train_df = pd.read_csv(".. /input/quora-insincere-questions-classification/train.csv")
 test_df = pd.read_csv(".. /input/quora-insincere-questions-classification/test.csv")
 train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_and_lower_text(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: clean_and_lower_text(x))
train_df["meta_number"] = train_df["question_text"].progress_apply(lambda x: have_pattern(x,number_re))
test_df["meta_number"] = test_df["question_text"].progress_apply(lambda x: have_pattern(x,number_re))
train_df["meta_mispel"] = train_df["question_text"].progress_apply(lambda x: have_pattern(x,mispellings_re))
test_df["meta_mispel"] = test_df["question_text"].progress_apply(lambda x: have_pattern(x,mispellings_re))
train_df["meta_words_nb"] = train_df["question_text"].progress_apply(lambda x : len(spaces_re.findall(x)))
 test_df["meta_words_nb"] = test_df["question_text"].progress_apply(lambda x : len(spaces_re.findall(x)))
 train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_numbers(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: clean_numbers(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
np.random.seed(SEED)
 trn_idx = np.random.permutation(len(train_X))
train_X = train_X[trn_idx]
train_y = train_y[trn_idx]
meta_train_X = train_df[[col for col in train_df if col.startswith('meta')]]
meta_test_X = test_df[[col for col in test_df if col.startswith('meta')]]
meta_train_X = preprocessing.scale(meta_train_X, with_mean='True', with_std='True')[trn_idx]
meta_test_X = preprocessing.scale(meta_test_X, with_mean='True', with_std='True')
 return train_X, test_X, train_y, tokenizer.word_index, meta_train_X, meta_test_X<save_model>
c=c1.drop('Cabin', axis=1 )
tqdm.pandas()
if not os.path.exists('./train_X.npy'):
start_time = time.time()
train_X, test_X, train_y, word_index, meta_train_X, meta_test_X = load_and_prec()
total_time =(time.time() - start_time)/ 60
print("Took {:.2f} minutes".format(total_time))

 else :
train_X = np.load('./train_X.npy', allow_pickle=True)
 test_X = np.load('./test_X.npy', allow_pickle=True)
 meta_train_X = np.load('./meta_train_X.npy', allow_pickle=True)
 meta_test_X = np.load('./meta_test_X.npy', allow_pickle=True)
 train_y = np.load('./train_y.npy', allow_pickle=True)
 with open('./word_index.pickle', 'rb')as handle:
tokenizer = pickle.load(handle)
 gc.collect()<split>
c['Age'].fillna(28, inplace=True )
def load_embedding(word_index, EMBEDDING_FILE):
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 if(EMBEDDING_FILE=='./embeddings/paragram_300_sl999/paragram_300_sl999.txt'):
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 else :
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index)+1)
 embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<load_pretrained>
c['Embarked'].fillna(method='ffill', inplace=True )
if not os.path.exists('./embedding_matrix.npy'):
start_time = time.time()
embedding_glove = load_embedding(word_index, './embeddings/glove.840B.300d/glove.840B.300d.txt')
 embedding_paragram = load_embedding(word_index, './embeddings/paragram_300_sl999/paragram_300_sl999.txt')
 embedding_matrix = np.mean([embedding_glove, embedding_paragram], axis=0)
 print(np.shape(embedding_matrix))
total_time =(time.time() - start_time)/ 60
print("Took {:.2f} minutes".format(total_time))
del embedding_glove, embedding_paragram
else :
embedding_matrix = np.load('./embedding_matrix.npy', allow_pickle=True)
 gc.collect()<init_hyperparams>
c['Fare'].fillna(method='ffill', inplace=True )
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )<choose_model_class>
print(c.isnull().sum() )
class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self ).__init__()
hidden_size = 60
self.embedding = nn.Embedding(np.shape(embedding_matrix)[0], embed_size, padding_idx=0)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.linear_meta = nn.Linear(meta_size, meta_size)
 self.embedding_dropout = nn.Dropout2d(0.1)
 self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm_attention = Attention(hidden_size*2, maxlen)
 self.gru_attention = Attention(hidden_size*2, maxlen)
 self.linear = nn.Linear(480, 16)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.softmax = nn.Softmax(dim=1)
 self.out = nn.Linear(16 + meta_size, 1)
 def forward(self, x, meta_x):
h_embedding = self.embedding(x)
 h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 h_lstm_atten = self.lstm_attention(h_lstm)
 h_gru_atten = self.gru_attention(h_gru)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 conc = torch.cat(( h_lstm_atten, h_gru_atten, avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
h_meta = self.linear_meta(meta_x.type(torch.float))
conc = torch.cat(( conc, h_meta), 1)
 conc = self.dropout(conc)
 out = self.out(conc)
 return out<compute_test_metric>
c2=c[['Pclass','Sex','Age','Embarked','family_size','Parch','SibSp', 'Fare']]
c2['Pclass']=c2['Pclass'].astype(object )
def sigmoid(x):
return 1 /(1 + np.exp(-x))<compute_test_metric>
c3=pd.get_dummies(c2)
 print("the shape of the original dataset",c2.shape)
 print("the shape of the encoded dataset",c3.shape)
 print("We have ",c3.shape[1]- c2.shape[1], 'new encoded features' )
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result<split>
Train = c3[:na]
Test = c3[na:]
splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED ).split(train_X, train_y))
train_preds = np.zeros(( len(train_X)))
 test_preds = np.zeros(( len(test_X)))
 seed_torch(SEED)
 x_test_cuda = torch.tensor(test_X, dtype=torch.long ).cuda()
x_meta_test_cuda = torch.tensor(meta_test_X, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda, x_meta_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(train_X[train_idx], dtype=torch.long ).cuda()
x_meta_train_fold = torch.tensor(meta_train_X[train_idx], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(train_X[valid_idx], dtype=torch.long ).cuda()
x_meta_val_fold = torch.tensor(meta_train_X[valid_idx], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()

 model = NeuralNet()
model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction="sum")
 optimizer = torch.optim.Adam(model.parameters())
 train = torch.utils.data.TensorDataset(x_train_fold, x_meta_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, x_meta_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(train_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for x_batch, meta_x_batch, y_batch in tqdm(train_loader, disable=True):
y_pred = model(x_batch, meta_x_batch)
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(len(test_X))
avg_val_loss = 0.
 for i,(x_batch, meta_x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch, meta_x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, train_epochs, avg_loss, avg_val_loss, elapsed_time))
for i,(x_batch, meta_x_batch)in enumerate(test_loader):
y_pred = model(x_batch, meta_x_batch ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits)
<compute_test_metric>
s=a[['Pclass','Fare','Survived']]
s.sort_values(by='Fare', ascending=False ).head(10 )
search_result = threshold_search(train_y, train_preds)
 search_result<save_to_csv>
Train1=Train[Train['Fare'] < 200]
sub = pd.read_csv('.. /input/quora-insincere-questions-classification/sample_submission.csv')
 sub.prediction = test_preds > search_result['threshold']
sub['prediction'] = sub['prediction'].apply(lambda x : int(x))
sub.to_csv("submission.csv", index=False )<set_options>
print('We dropped ',Train.shape[0] - Train1.shape[0],'fare outliers' )
warnings.filterwarnings('ignore' )<load_from_csv>
Train1['SibSp'].sort_values(ascending=False ).head(7 )
quora_train = pd.read_csv("/kaggle/input/quora-insincere-questions-classification/train.csv")
 quora_test = pd.read_csv("/kaggle/input/quora-insincere-questions-classification/test.csv")
 quora_train.head(1 )<create_dataframe>
train=Train1[Train1['SibSp'] <= 5]
paragramModel = Embeddings(file_path,file="paragram" )<categorify>
print('We dropped ', Train1.shape[0]- train.shape[0], 'sibSp outliers')
 print('And in total, we dropped ', Train.shape[0]-train.shape[0], 'outliers' )
def str_lowercase_text(text):
text = text.map(str)
 for line in range(len(text.values)) :
text.values[line] = text.values[line].lower()
return text
quora_train['question_text_paragram'] = str_lowercase_text(quora_train['question_text'])
 quora_test['question_text_paragram'] = str_lowercase_text(quora_test['question_text'])
 print("str_lowercase_text : Done")
 replacingWords = dict({"√":" sqrt ","π":" pi ","α":" alpha ","θ":" theta ","∞":" infinity ",
"∝":" proportional to ","sinx":" sin x ","cosx":" cos x ", "tanx":" tan x ","cotx":" cot x ",
"secx":" sec x ", "cosecx":" cosec x ", "£":" pound ", "β":" beta ", "σ": " theta ", "∆":" delta ",
"μ":" mu ",'∫': " integration ", "ρ":" rho ", "λ":" lambda ","∩":" intersection ","Δ":" delta ",
"φ":" phi ", "℃":" centigrade ","≠":" does not equal to ","Ω":" omega ","∑":" summation ",
"∪":" union ", "ψ":" psi ", "Γ":" gamma ","⇒":" implies ","∈":" is an element of ",
"≡":" is congruent to ", "≈":" is approximately equal to ", "~":" is distributed as ",
"≅":" is isomorphic to ","⩽":" is less than or equal to ","≥":" is greater than or equal to ",
"⇐":" is implied by ","⇔":" is equivalent to ", "∉":" is not an element of ","∅" : " empty set ",
"∛":"cube root","÷":" division ","㏒":" log ","∇":" del ","⊆":" is a subset of ","±":" plus–minus ",
"⊂":" is a proper subset of ","€":" euro ","㏑":" ln ","₹":" rupee ","∀":" there exists ","∛":"cube root",
"⅓":" one by three ","½":" one by two ","∈":" is an element of ","¼":" one by four "})
 def special_chars(text,symbols):
for p in symbols:
if p in text:
text = text.replace(p, symbols[p])
 text = re.sub("\s{2}"," ",text)
 return text
quora_train['question_text_paragram'] = quora_train['question_text_paragram'].apply(lambda x: special_chars(x,replacingWords))
quora_test['question_text_paragram'] = quora_test['question_text_paragram'].apply(lambda x: special_chars(x,replacingWords))
print("special_chars : Done")
 characters = "＝ా̫̾̀ͅ⚧ਿ∖⁡્⬇☉ూिାੁ͔☺͛ॢി「̷̊̆﻿َ«̰︡？◦✏ូ‬͒ِ℅„〖ௌ•­‐̗∧̯িֿ̔〗்“·้″∂͚̑ी∴ు́̕♡¯❓̦̓ை₊ுं”‌☁×ొ、್⌚​—’̶̋̐⎝ैು¸̞͑（⋅ृ′͋‘়͊➡†ী️ா̥：̻ू∗＾\
→´្ाাோ✌。⊥̵̛¬̒ാ–»！∨・❤̝ంి̮⎞」͆।☝่̙˚̬͌͘¦്，̴̂˜ੀ│ីৃ⃗᠌¡ੰ̧♀✔̈́̓）⊨✅￼⎛ិ↓्̉ुॣ；ّุ∡̭∘－؟△⋯ॉॄ✓∠̲̺®‏♭̱̍ു్̹̌̚͜ौ⁻ె⟨͈́⎠ँ™͝ំಿ¢̿ં͠↑،ើ̪ਂ₩̄̎‑̢ಾ¾₱̃ো︠ਾ≱͖ः¨⁠ાி͡ে਼ៃ̣͂》♏̜\
̜§͕‪《͎≤̇〇／ْ₦̼¥▾…ះี̖̘͇̽͐ា‛♨‰़̤̳̅⧽−ೋ̀▒ٌู̩⅔♣̡ั∼͉¿͗°☹⟩্©¶ो˂＞े◌‎̸ీ⧼＄̟̈⦁"
character_list = ['^', ')', '@', ',', '$', '+', '/', '?', '"', ';', '[', '%', '*', ']', "'", '>', '|', '=', '<', '.',
'&', '`', '\', '
def remove_punct(text,punToBeRemove):
translate_table = dict(( ord(char), None)for char in punToBeRemove)
 for idx,val in(enumerate(text.values)) :
val = val.translate(translate_table)
 text.values[idx] = val.strip()
return text
quora_train['question_text_paragram'] = remove_punct(quora_train['question_text_paragram'], characters)
 quora_test['question_text_paragram'] = remove_punct(quora_test['question_text_paragram'], characters)
 print("remove_punct : Done")
 contractions = {"'aight": 'alright', "ain't": 'am not', "amn't": 'am not', "aren't": 'are not', "can't": 'can not',
"'cause": 'because', "could've": 'could have', "couldn't": 'could not', "couldn't've": 'could not have', "daren't":
'dare not', "daresn't": 'dare not', "dasn't": 'dare not', "didn't": 'did not', "doesn't": 'does not',
"don't": 'do not', 'dunno': "don't know", "d'ye": 'do you', "e'er": 'ever', "everybody's": 'everybody is',
"everyone's": 'everyone is', 'finna': 'fixing to', "g'day": 'good day', 'gimme': 'give me', "giv'n": 'given',
'gonna': 'going to', "gon't": 'go not', 'gotta': 'got to', "hadn't": 'had not', "had've": 'had have',
"hasn't": 'has not', "haven't": 'have not', "he'd": 'he had', "he'll": 'he will', "he's": 'he is',
"he've": 'he have', "how'd": 'how did', 'howdy': 'how do you do', "how'll": 'how will', "how're": 'how are',
"how's": 'how is', "I'd": 'I had', "I'd've": 'I would have', "I'll": 'I will', "I'm": 'I am',
"I'm'a": 'I am about to', "I'm'o": 'I am going to', 'innit': 'is it not', "I've": 'I have', "isn't": 'is not',
"it'd ": 'it would', "it'll": 'it will', "it's ": 'it is', 'iunno': "I don't know", "let's": 'let us',
"ma'am": 'madam', "mayn't": 'may not', "may've": 'may have', 'methinks': 'me thinks', "mightn't": 'might not',
"might've": 'might have', "mustn't": 'must not', "mustn't've": 'must not have', "must've": 'must have',
"needn't": 'need not', 'nal': 'and all', "ne'er": 'never', "o'clock": 'of the clock', "o'er": 'over',
"ol'": 'old', "oughtn't": 'ought not', "'s": 'is', "shalln't": 'shall not', "shan't": 'shall not',
"she'd": 'she would', "she'll": 'she will', "she's": 'she is', "should've": 'should have',
"shouldn't": 'should not', "shouldn't've": 'should not have', "somebody's": 'somebody has',
"someone's": 'someone has', "something's": 'something has', "so're": 'so are', "that'll": 'that will',
"that're": 'that are', "that's": 'that is', "that'd": 'that would', "there'd": 'there would',
"there'll": 'there will', "there're": 'there are', "there's": 'there is', "these're": 'these are',
"they've": 'they have', "this's": 'this is', "those're": 'those are', "those've": 'those have', "'tis": 'it is',
"to've": 'to have', "'twas": 'it was', 'wanna': 'want to', "wasn't": 'was not', "we'd": 'we would',
"we'd've": 'we would have', "we'll": 'we will', "we're": 'we are', "we've": 'we have', "weren't": 'were not',
"what'd": 'what did', "what'll": 'what will', "what're": 'what are', "what's": 'what does', "what've": 'what have',
"when's": 'when is', "where'd": 'where did', "where'll": 'where will', "where're": 'where are',
"where's": 'where is',"where've": 'where have', "which'd": 'which would', "which'll": 'which will',
"which're": 'which are',"which's": 'which is', "which've": 'which have', "who'd": 'who would',
"who'd've": 'who would have', "who'll": 'who will', "who're": 'who are', "who'ves": 'who is', "who'": 'who have',
"why'd": 'why did', "why're": 'why are', "why's": 'why does', "willn't": 'will not', "won't": 'will not',
'wonnot': 'will not', "would've": 'would have', "wouldn't": 'would not', "wouldn't've": 'would not have',
"y'all": 'you all', "y'all'd've": 'you all would have', "y'all'd'n've": 'you all would not have',
"y'all're": 'you all are', "cause":"because","have't":"have not","cann't":"can not","ain't":"am not",
"you'd": 'you would', "you'll": 'you will', "you're": 'you are', "you've": 'you have', 'cannot': 'can not',
'wont': 'will not', "You'": 'Am not', "Ain'": 'Am not', "Amn'": 'Am not', "Aren'": 'Are not',
"Can'": 'Because', "Could'": 'Could have', "Couldn'": 'Could not have', "Daren'": 'Dare not',
"Daresn'": 'Dare not', "Dasn'": 'Dare not', "Didn'": 'Did not', "Doesn'": 'Does not', "Don'": "Don't know",
"D'": 'Do you', "E'": 'Ever', "Everybody'": 'Everybody is', "Everyone'": 'Fixing to', "G'": 'Give me',
"Giv'": 'Going to', "Gon'": 'Got to', "Hadn'": 'Had not', "Had'": 'Had have', "Hasn'": 'Has not',
"Haven'": 'Have not', "He'": 'He have', "How'": 'How is', "I'": 'I have', "Isn'": 'Is not', "It'": "I don't know",
"Let'": 'Let us', "Ma'": 'Madam', "Mayn'": 'May not', "May'": 'Me thinks', "Mightn'": 'Might not',
"Might'": 'Might have', "Mustn'": 'Must not have', "Must'": 'Must have', "Needn'": 'And all', "Ne'": 'Never',
"O'": 'Old', "Oughtn'": 'Is', "Shalln'": 'Shall not', "Shan'": 'Shall not', "She'": 'She is',
"Should'": 'Should have', "Shouldn'": 'Should not have', "Somebody'": 'Somebody has', "Someone'": 'Someone has',
"Something'": 'Something has', "So'": 'So are', "That'": 'That would', "There'": 'There is',
"They'": 'They have', "This'": 'This is', "Those'": 'It is', "To'": 'Want to', "Wasn'": 'Was not',
"Weren'": 'Were not', "What'": 'What have', "When'": 'When is', "Where'": 'Where have', "Which'": 'Which have',
"Who'": 'Who have', "Why'": 'Why does', "Willn'": 'Will not', "Won'": 'Will not', "Would'": 'Would have',
"Wouldn'": 'Would not have', "Y'": 'You all are',"What's":"What is","What're":"What are","what's":"what is",
"what're":"what are", "Who're":"Who are", "your're":"you are","you're":"you are", "You're":"You are",
"We're":"We are", "These'": 'These have', "we're":"we are","Why're":"Why are","How're":"How are ",
"how're ":"how are ","they're ":"they are ", "befo're":"before","'re ":" are ",'don"t ':"do not",
"Won't ":"Will not ","could't":"could not", "would't":"would not", "We'": 'We have',"Hasn't":"Has not",
"n't":"not", 'who"s':"who is"}
def decontraction(text,contractions):
for idx,val in enumerate(text.values):
val = ' '.join(word.replace(word,contractions[word])if word in contractions
else word for word in val.split())
 val = re.sub(r"'s", " ", val);val = re.sub(r"''s", " ", val);val = re.sub(r""s", " ", val)
 val = re.sub(r"n't", " not ", val);val = re.sub(r"n''t", " not ", val);val = re.sub(r"n"t", " not ", val)
 val = re.sub(r"'re ", " are ", val);val = re.sub(r"'d ", " would", val);val = re.sub(r"''d ", " would", val)
 val = re.sub(r""d ", " would", val);val = re.sub(r"'ll ", " will", val);val = re.sub(r"''ll ", " will", val)
 val = re.sub(r""ll ", " will", val);val = re.sub(r"'ve ", " have", val);val = re.sub(r"''ve ", " have", val)
 val = re.sub(r""ve ", " have", val);val = re.sub(r"'m ", " am", val);val = re.sub(r"''m "," am", val)
 val = re.sub(r""m "," am", val);val = re.sub("\s{2}"," ",val)
 text.values[idx] = val.strip()
return text

 def spacing_of_chars(text,characters_list):
for char in characters_list:
if char in text:
text = text.replace(char," "+char+" ")
 text = re.sub("\s+"," ",text)
 return text
quora_train['question_text_paragram'] = quora_train['question_text_paragram'].apply(lambda x: spacing_of_chars(x,character_list))
quora_test['question_text_paragram'] = quora_test['question_text_paragram'].apply(lambda x: spacing_of_chars(x,character_list))
print("spacing_of_chars : Done")
 def coverage(vocab, embeddings_index,print_statement=False):
known_words = defaultdict(int)
 unknown_words = defaultdict(int)
 knownWordsVal = 0
unknownWordsVal = 0
for word in vocab.keys() :
try:
known_words[word] = embeddings_index[word]
knownWordsVal += vocab[word]
except:
unknown_words[word] = vocab[word]
unknownWordsVal += vocab[word]
pass
if print_statement == True:
print('Found {:.2%} of words in the embedding of the question text vocab' 
.format(len(known_words)/ len(vocab)))
 print('Found {:.2%} of the words in the question text vocab'.format(knownWordsVal /(knownWordsVal + unknownWordsVal)))
 else:
pass
unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words
def question_text_vocab(text):
freq_dict = defaultdict(int)
 total_sent = text.apply(lambda x: x.split() ).values
for sent in total_sent:
for token in sent:
freq_dict[token] += 1
return freq_dict
TextVocabTrain = question_text_vocab(quora_train['question_text_paragram'])
 TextVocabTest = question_text_vocab(quora_test['question_text_paragram'])
 print("question_text_vocab : Done")
 OOVGloveTrain = coverage(TextVocabTrain, paragramModel)
 OOVGloveTest = coverage(TextVocabTest, paragramModel)
 print("coverage : Done")
 def check_oov_vocab(vocab, embeddings):
freq_dict = defaultdict()
for word in vocab:
if word[0].istitle() == True:
if word[0].lower() in embeddings:
freq_dict[word[0]]= word[0].lower()
elif word[0].islower() == True:
if word[0].title() in embeddings:
freq_dict[word[0]]= word[0].title()
return freq_dict
WordDictTrain = check_oov_vocab(OOVGloveTrain,paragramModel)
 WordDictTest = check_oov_vocab(OOVGloveTest,paragramModel)
 print("check_oov_vocab : Done")
 def replace_words(text,set_words):
for idx,val in enumerate(text.values):
val = ' '.join(word.replace(word,set_words[word])if word in set_words else word for word in val.split())
 text.values[idx] = val
return text
quora_train['question_text_paragram'] = replace_words(quora_train['question_text_paragram'], WordDictTrain)
 quora_test['question_text_paragram'] = replace_words(quora_test['question_text_paragram'], WordDictTest)
 print("replace_words : Done")
 replace_word = dict({"quorans":"quora", "brexit":"britain exit", "cryptocurrencies":"cryptocurrency",
"blockchain":"blockchain", "demonetisation":"demonetization", "pokémon":"pokemon",
"qoura":"quora", "fiancé":"fiance","cryptocurrency":"cryptocurrency", "x²":"x squeare",
"quoras":"quora","whst":"what", "²":"square", "Demonetization":"demonetization",
"brexit":"britain exit", "são":"sao","genderfluid":"Gender fluid", "howcan":"How can",
"undergraduation":"under graduation", "whydo":"why do", "à":"a","chapterwise":"chapter wise",
"cryptocurrencies":"cryptocurrency", "fiancée":"fiance", "wouldwin":"would win",
"nanodegree":"nano degree","nanodegree":"nano degree", "blockchains":"blockchain",
"clichés":"cliche", "erdoğan":"erdogan", "beyoncé":"beyonce", "fullform":"full form",
"atatürk":"ataturk", "Whyis":"Why is","amfrom":"am from", "2k17":"2017",
"demonitization":"demonetization", "cliché":"cliche", "montréal":"montreal",
"thé":"the", "am17":"am 17", "willhappen":"will happen","³":"cube", "whatapp":"whatsapp",
"ε":"epsilon", "whatsaap":"whatsapp",'Σ':"summation","quorians":"quora users",
"cryptocurreny":"cryptocurrency", "mastuburation":"masturbation","Whatre":"What are",
"whatdo":"what do","δ":"delta","oversmart":"over smart","¹":"one","baahubali":"baahubali",
"note4":"note 4", "gdpr":"general data protection regulation", "bnbr":"' be nice , be respectful '",
"uceed":"undergraduate common entrance examination for design","bhakts":"bhakts",
"iiest":"indian institutes of engineering science and technology","bhakths":"bhakts",
"upwork":"Upwork","unacademy":"Unacademy","squeare":"square","srmjeee":"srmjee",
"demonitisation":"demonetization",
"cos2x":"cos 2x","padmavat":"padmaavat", "flipcart":"flipkart",
"havegot":"have got","2k18":"2018","a²":"a square","whydoes":"why does","sina":"sin a",
"class9":"class 9"})
 quora_train['question_text_paragram'] = replace_words(quora_train['question_text_paragram'], replace_word)
 quora_test['question_text_paragram'] = replace_words(quora_test['question_text_paragram'], replace_word)
 print("replace_words : Done" )<import_modules>
x=train
y=np.array(target )
import tensorflow as tf
from tensorflow.keras import backend as K
from tensorflow.keras.backend import clear_session, maximum
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.layers import Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D
from tensorflow.keras import Model, initializers, regularizers, constraints, optimizers, layers
from tensorflow.keras.regularizers import l2
from tensorflow.keras.layers import Dense, Input, Dropout, Conv1D, MaxPooling1D, Flatten, Embedding
from tensorflow.keras.layers import Concatenate, LSTM, Activation, GRU, Reshape, Lambda, Multiply, Bidirectional, Maximum
from tensorflow.keras.preprocessing.text import Tokenizer<split>
x_train, x_test, y_train, y_test = train_test_split(x, y,test_size =.33, random_state=0 )
y = quora_train['target']
X = quora_train.drop(columns = ['target'])
 X_train, X_cv, y_train, y_cv = train_test_split(X, y, test_size=0.20, stratify=y)
 print("The shape of train,cv & test dataset before conversion into vector")
 print(X_train.shape, y_train.shape)
 print(X_cv.shape, y_cv.shape)
 print(quora_test.shape )<define_variables>
scaler= StandardScaler()
x_train = scaler.fit_transform(x_train)
 x_test = scaler.transform(x_test)
 test = scaler.transform(Test )
maxlength = 75
embedding_dim = 300<categorify>
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import roc_curve, auc
from sklearn.metrics import classification_report
def TokenizationPadding(data, maxlen):
encoder_data = list()
tokens = Tokenizer()
tokens.fit_on_texts(data[0])
 for idx,val in enumerate(data):
encoder_data.append(tokens.texts_to_sequences(val))
vocab_size = len(tokens.word_index)+1
seq_padding = list()
for val in encoder_data:
seq_padding.append(pad_sequences(val, maxlen=maxlen, padding='post'))
return seq_padding, vocab_size, tokens<create_dataframe>
lr_c=LogisticRegression(C=1, class_weight={0:0.62, 1:0.38}, max_iter=5000,
penalty='l2',
random_state=None, solver='lbfgs', verbose=0,
warm_start=True)
 lr_c.fit(x_train,y_train.ravel())
 lr_pred=lr_c.predict(x_test)
 lr_ac=accuracy_score(y_test.ravel() , lr_pred)
 print('LogisticRegression_accuracy test:',lr_ac)
 print("AUC",roc_auc_score(y_test.ravel() , lr_pred))
data = [X_train["question_text_paragram"], X_cv["question_text_paragram"], quora_test["question_text_paragram"]]<statistical_test>
cv = StratifiedShuffleSplit(n_splits = 15, test_size =.25, random_state = 0)
 x = scaler.fit_transform(x)
 accuracies = cross_val_score(LogisticRegression(solver='liblinear',class_weight={0:0.62, 1:0.38}), x,y, cv = cv)
 print("CV accuracy on 15 chunks: {}".format(accuracies))
print("Mean CV accuracy: {}".format(round(accuracies.mean() ,5)) )
seq_padding, vocab_size, tokenizer = TokenizationPadding(data, maxlength)
 Xtrain, Xcv, Xtest = seq_padding[0], seq_padding[1], seq_padding[2]<categorify>
svm=SVC(kernel='rbf',C=10,gamma=0.1)
 svm.fit(x_train,y_train.ravel())
 svm_pred=svm.predict(x_test)
 print('Accuracy is ',metrics.accuracy_score(svm_pred,y_test.ravel()))
print('AUC: ',roc_auc_score(y_test.ravel() , svm_pred))
def embedding_matrix(vocab_size,model,dim,tokenizer):
keys = set(model.keys())
 emb_matrix = np.zeros(( vocab_size,dim))
for idx,val in tokenizer.word_index.items() :
if idx in keys:
emb_vector = model[idx]
emb_matrix[val] = emb_vector
print('The shape of emdedding matrix is: ',emb_matrix.shape)
 return emb_matrix<load_pretrained>
SVMC = SVC(probability=True,class_weight={0:0.62, 1:0.38})
 svc_param_grid = {'kernel': ['rbf'],
'gamma': [ 0.001,0.008, 0.01,0.02,0.05, 0.1, 1],
'C': [1, 10, 12, 14, 20, 25, 30, 40,50,60, 100,200,300, 1000]}
gsSVMC = GridSearchCV(SVMC,param_grid = svc_param_grid, cv=5, scoring="accuracy", n_jobs=-1)
 gsSVMC.fit(x_train,y_train)
 SVMC_best = gsSVMC.best_estimator_
SVMC_best.fit(x_train, y_train.ravel())
 pred_svm = SVMC_best.predict(x_test)
 acc_svm = accuracy_score(y_test.ravel() , pred_svm)
 print('Accuracy', acc_svm)
 print('AUC: ',roc_auc_score(y_test.ravel() , pred_svm))
print(SVMC_best )
embedding_matrix_paragram = embedding_matrix(vocab_size, paragramModel, embedding_dim, tokenizer )<categorify>
rdf_c=RandomForestClassifier(n_estimators=100,criterion='entropy',random_state=0)
 rdf_c.fit(x_train,y_train.ravel())
 rdf_pred=rdf_c.predict(x_test)
 rdf_ac=accuracy_score(rdf_pred,y_test.ravel())
 print('Accuracy of random forrest classifier: ',rdf_ac)
 print('AUC: ',roc_auc_score(y_test.ravel() , rdf_pred))
ytrain = to_categorical(y_train, 2)
 ycv = to_categorical(y_cv, 2 )<prepare_x_and_y>
knn_clf = KNeighborsClassifier()
parameters_knn = {"n_neighbors": [3, 5, 10, 15], "weights": ["uniform", "distance"], "algorithm": ["auto", "ball_tree", "kd_tree"],
"leaf_size": [20, 30, 50]}
grid_knn = GridSearchCV(knn_clf, parameters_knn, scoring='accuracy', cv=5, n_jobs=-1)
 grid_knn.fit(x_train, y_train)
 knn_clf = grid_knn.best_estimator_
knn_clf.fit(x_train, y_train.ravel())
 pred_knn = knn_clf.predict(x_test)
 acc_knn = accuracy_score(y_test.ravel() , pred_knn)
 print("The accuracy of KNeighbors is: " + str(acc_knn))
print('AUC: ',roc_auc_score(y_test.ravel() , pred_knn))
class accuracy_value(Callback):
def __init__(self,training_data,validation_data):
self.X_train = training_data[0]
self.y_train = training_data[1]
self.X_val = validation_data[0]
self.y_val = validation_data[1]
def on_train_begin(self, logs = {}):
self.f1_scores = []
self.precisions = []
self.recalls = []
def on_epoch_end(self, epoch, logs = {}):
y_predicted = np.asarray(self.model.predict(self.X_val)).round()
f1_val = f1_score(self.y_val,y_predicted,average=None)
 self.f1_scores.append(f1_val)
 print(" - f1 score : {}".format(np.round(f1_val,4)))
 f1Score = accuracy_value(training_data=(Xtrain, ytrain), validation_data=(Xcv, ycv))<choose_model_class>
xg_clf = XGBClassifier()
parameters_xg = {"objective" : ["reg:linear"], "n_estimators" : [5, 10, 15, 20]}
grid_xg = GridSearchCV(xg_clf, parameters_xg, scoring='accuracy',cv=5,n_jobs=-1)
 grid_xg.fit(x_train, y_train)
 xg_clf = grid_xg.best_estimator_
xg_clf.fit(x_train, y_train.ravel())
 pred_xg = xg_clf.predict(x_test)
 acc_xg = accuracy_score(y_test.ravel() , pred_xg)
 print("The accuracy of XGBoost is: " + str(acc_xg))
print('AUC: ',roc_auc_score(y_test.ravel() , pred_xg))
earlyStopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=0, verbose=0, mode='auto' )<choose_model_class>
nr.seed(1115)
 nn_mod = MLPClassifier(hidden_layer_sizes =(100,100,), max_iter=1750, solver='sgd')
 nn_mod.fit(x_train, y_train.ravel())
 scores = nn_mod.predict(x_test)
 nnacc = accuracy_score(y_test.ravel() , scores)
 print('The accuracy of MLP classifier: ',nnacc)
 print('AUC: ',roc_auc_score(y_test.ravel() , scores))
class Attention(tf.keras.layers.Layer):

 def __init__(self, att_units):
super(Attention, self ).__init__()
self.att_units = att_units
self.W1=tf.keras.layers.Dense(att_units)
 self.W2=tf.keras.layers.Dense(att_units)
 self.V=tf.keras.layers.Dense(1)
 def call(self,lstm_output, hidden_state):

 state_with_time_axis = tf.expand_dims(hidden_state, 1)
 score = self.V(tf.nn.tanh(self.W1(state_with_time_axis)+ self.W2(lstm_output)))
 attention_weights=tf.nn.softmax(score, axis=1)
 context_vector = attention_weights*lstm_output
context_vector = tf.reduce_sum(context_vector, axis=1)
 return context_vector, attention_weights<choose_model_class>
Bagg_estimators = [10,25,50,75,100,150,250];
cv = StratifiedShuffleSplit(n_splits=10, test_size=.33, random_state=15)
 parameters = {'n_estimators':Bagg_estimators }
gridBG = GridSearchCV(BaggingClassifier(base_estimator= None,
bootstrap_features=False),
param_grid=parameters,
cv=cv,
n_jobs = -1)
 bg_mod=gridBG.fit(x_train,y_train.ravel())
 bg_pred = bg_mod.predict(x_test)
 bg_acc = accuracy_score(y_test.ravel() , bg_pred)
 print("Bagging accuracy: ", bg_acc)
 print('AUC: ',roc_auc_score(y_test.ravel() , bg_pred))
clear_session()
inputs = Input(shape=(maxlength,), dtype='int32', name='Input_Text')
 Embedding_Layer = Embedding(vocab_size, 300, weights=[embedding_matrix_paragram], input_length=maxlength, trainable=False )(inputs)
 lstm_output, fw_state_h, fw_state_c, bw_state_h, bw_state_c = Bidirectional(LSTM(64, return_sequences=True, return_state=True))(Embedding_Layer)
 state_h = Concatenate()([fw_state_h, bw_state_h])
 context_vector, attention_weights = Attention(10 )(lstm_output, state_h)
 dense = Dense(64, activation='relu', name= "Dense_Layer" )(context_vector)
 output = Dense(2, activation='sigmoid', name= "Output_Layer" )(dense)
 model = Model(inputs,output)
 model.compile(loss = "binary_crossentropy", optimizer = "adam", metrics = ["accuracy"])
 model.fit(Xtrain, ytrain, batch_size=512, verbose=1, epochs=10,validation_data=(Xcv,ycv), shuffle=True, callbacks=[f1Score, earlyStopping] )<find_best_params>
n_estimators = [100,140,145,150,160, 170,175,180,185];
cv = StratifiedShuffleSplit(n_splits=10, test_size=.30, random_state=15)
 learning_rate = [0.1,1,0.01,0.5]
parameters = {'n_estimators':n_estimators,
'learning_rate':learning_rate}
gridAda = GridSearchCV(AdaBoostClassifier(base_estimator= None, 
),
param_grid=parameters,
cv=cv,
n_jobs = -1)
 ada_mod= gridAda.fit(x_train,y_train.ravel())
 ada_pred = ada_mod.predict(x_test)
 ada_acc = accuracy_score(y_test.ravel() , ada_pred)
 print("Adaboost accuracy: ", ada_acc)
 print('AUC: ',roc_auc_score(y_test.ravel() , ada_pred))
threshold = dict()
ypred = model.predict(Xcv, batch_size=512,verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.05):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, round(list(metrics.f1_score(ycv,(ypred>thresh ).astype(int), average=None)) [1],3)))
 threshold[thresh] = round(list(metrics.f1_score(ycv,(ypred>thresh ).astype(int), average=None)) [1],3)
 print("
The best threshold is:",max(threshold, key=threshold.get))
ypredicted =(ypred>max(threshold, key=threshold.get)).astype(int)
 print("Classification Report:
",metrics.classification_report(ycv,ypredicted))<predict_on_test>
gradient_boost = GradientBoostingClassifier()
gradient_boost.fit(x_train, y_train.ravel())
 ygbc_pred = gradient_boost.predict(x_test)
 gradient_accy = round(accuracy_score(ygbc_pred, y_test.ravel()), 3)
 print('Gradient Boosting accuracy: ',gradient_accy)
 print('AUC: ',roc_auc_score(y_test.ravel() , ygbc_pred))
ypredict = list()
ypred = model.predict(Xtest, batch_size=512,verbose=1)
 for i in ypred:
ypredict.append(( i[1]>max(threshold, key=threshold.get)).astype(int))
df_test = pd.DataFrame({"qid":quora_test["qid"].values})
 df_test['prediction'] = ypredict
print("Quora Test Output:
",df_test['prediction'].value_counts() )<save_to_csv>
ExtraTreesClassifier = ExtraTreesClassifier()
ExtraTreesClassifier.fit(x_train, y_train.ravel())
 y_pred = ExtraTreesClassifier.predict(x_test)
 extraTree_accy = accuracy_score(y_pred, y_test.ravel())
 print('ExtraTrees classifier accuracy: ',extraTree_accy)
 print('AUC: ',roc_auc_score(y_test.ravel() , y_pred))
df_test.to_csv('submission.csv', index=False )<import_modules>
GaussianProcessClassifier = GaussianProcessClassifier()
GaussianProcessClassifier.fit(x_train, y_train.ravel())
 yg_pred = GaussianProcessClassifier.predict(x_test)
 gau_pro_accy = accuracy_score(yg_pred, y_test.ravel())
 print('Gaussian process classifier: ', gau_pro_accy)
 print('AUC: ',roc_auc_score(y_test.ravel() , yg_pred))
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.layers import Dense, Input, Embedding, Dropout, Activation, Conv1D
from tensorflow.keras.layers import GRU,LSTM
from tensorflow.keras.layers import Bidirectional, GlobalMaxPooling1D, MaxPool1D
from tensorflow.keras.models import Model , Sequential
from tensorflow.keras import initializers, regularizers, constraints, optimizers, layers
from tensorflow.keras.callbacks import EarlyStopping<load_from_csv>
voting_classifier = VotingClassifier(estimators=[ 
('gradient_boosting', gradient_boost), 
('bagging_classifier', bg_mod), 
('ada_classifier',ada_mod), 
('XGB_Classifier', xg_clf), 
('gaussian_process_classifier', GaussianProcessClassifier)
 ],voting='hard')
 voting_classifier = voting_classifier.fit(x_train,y_train.ravel() )
df_train=pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv' )<load_from_csv>
vote_pred = voting_classifier.predict(x_test)
 voting_accy = round(accuracy_score(vote_pred, y_test.ravel()), 4)
 print('Voting accuracy of the combined classifiers: ',voting_accy)
 print('AUC: ',round(roc_auc_score(y_test.ravel() , vote_pred), 4))
df_test=pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv' )<import_modules>
print(classification_report(y_test, vote_pred))
from zipfile import ZipFile 
<load_pretrained>
final_pred = voting_classifier.predict(test )
dim=300
embeddings1_index={}
with zipfile.ZipFile(".. /input/quora-insincere-questions-classification/embeddings.zip")as zf:
with io.TextIOWrapper(zf.open("glove.840B.300d/glove.840B.300d.txt"), encoding="utf-8")as f:
for line in tqdm(f):
values=line.split(' ')
 word=values[0]
vectors=np.asarray(values[1:],'float32')
 embeddings1_index[word]=vectors<train_model>
titanic_submission = pd.DataFrame({
"PassengerId": b["PassengerId"],
"Survived": final_pred
})
 titanic_submission.to_csv("titanic.csv", index=False )