Kaggle competition: Quora Insincere Questions Classification
def cabin_imputer(cabin):
if cabin != "Unknown":
return cabin[0]
return cabin
def age_to_cat(age):
if np.isnan(age):
return "Unknown"
elif age < 13:
return "Kid"
elif age <= 18:
return "Teen"
elif age > 60:
return "Elder"
else:
return "Adult"
def substrings_in_string(big_string, substrings):
for substring in substrings:
if big_string.find(substring)!= -1:
return substring
print(big_string)
 return np.nan
def replace_titles(x):
title=x['Title']
if title in ['Don', 'Major', 'Capt', 'Jonkheer', 'Rev', 'Col']:
return 'Mr'
elif title in ['Countess', 'Mme']:
return 'Mrs'
elif title in ['Mlle', 'Ms']:
return 'Miss'
elif title =='Dr':
return "Dr"
else:
return title
def clear_dataset(dataset):
dataset["Family"] = dataset["Parch"] + dataset["SibSp"] + 1
dataset["Alone"] = dataset["Family"] == 1
dataset.Cabin = dataset.Cabin.fillna("Unknown")
 dataset["Cabin"] = dataset["Cabin"].apply(cabin_imputer)
 dataset.Embarked = dataset.Embarked.fillna("S")
 dataset.Fare = dataset.groupby(by="Pclass")["Fare"].apply(lambda x: x.fillna(x.mean()))
dataset["Age_Cat"] = dataset.Age.apply(age_to_cat)
 dataset["Age"] = dataset.Age.fillna(dataset.Age.mean())
 title_list=['Mrs', 'Mr', 'Master', 'Miss', 'Major', 'Rev',
'Dr', 'Ms', 'Mlle','Col', 'Capt', 'Mme', 'Countess',
'Don', 'Jonkheer']
dataset['Title']=dataset['Name'].map(lambda x: substrings_in_string(x, title_list))
dataset['Title']=dataset.apply(replace_titles, axis=1)
 dataset["FareLog"] = dataset.Fare.apply(np.log1p)
 dataset["FarePerFamilyMember"] = dataset.Fare / dataset.Family
dataset.drop(["Ticket","Name"], axis=1, inplace=True)
 return dataset<load_from_csv>
%matplotlib inline
pd.set_option('max_colwidth',400)
 warnings.filterwarnings("ignore", message="F-score is ill-defined and being set to 0.0 due to no predicted samples.")
data_train = pd.read_csv(".. /input/train.csv")
 data_test = pd.read_csv(".. /input/test.csv")
 y_data = data_train.Survived
x_data_train = clear_dataset(data_train)
 x_data_test = clear_dataset(data_test )<drop_column>
def seed_torch(seed=1029):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
x_data_train.drop(["PassengerId", "Survived"], axis=1,inplace=True)
 x_data_test.drop(["PassengerId"], axis=1,inplace=True )<categorify>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 sub = pd.read_csv('.. /input/sample_submission.csv' )
test_encoded = pd.get_dummies(x_data_test)
 train_encoded = pd.get_dummies(x_data_train)
 test_encoded= test_encoded.reindex(columns = train_encoded.columns, fill_value=0 )<normalization>
train["target"].value_counts()
pca = PCA(n_components = 2, whiten= True)
 x_pca = pca.fit_transform(train_encoded)
 print("variance ratio: ", pca.explained_variance_ratio_)
 print("sum: ",sum(pca.explained_variance_ratio_))<split>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"that's" : "that is",
"there's" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not",
"tryin'":"trying"}
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
 train["question_text"] = train["question_text"].apply(lambda x: clean_text(x.lower()))
test["question_text"] = test["question_text"].apply(lambda x: clean_text(x.lower()))
train["question_text"] = train["question_text"].apply(lambda x: clean_numbers(x))
test["question_text"] = test["question_text"].apply(lambda x: clean_numbers(x))
train["question_text"] = train["question_text"].apply(lambda x: replace_typical_misspell(x))
test["question_text"] = test["question_text"].apply(lambda x: replace_typical_misspell(x))
x_train, x_val, y_train, y_val = train_test_split(train_encoded, y_data, test_size=0.25, random_state=42 )<compute_test_metric>
max_features = 120000
tk = Tokenizer(lower = True, filters='', num_words=max_features)
 full_text = list(train['question_text'].values)+ list(test['question_text'].values)
 tk.fit_on_texts(full_text )
def get_metrics(y_test, y_predicted):
precision = precision_score(y_test, y_predicted, pos_label=None,
average='weighted')
 recall = recall_score(y_test, y_predicted, pos_label=None,
average='weighted')
 f1 = f1_score(y_test, y_predicted, pos_label=None, average='weighted')
 accuracy = accuracy_score(y_test, y_predicted)
 return accuracy, precision, recall, f1<compute_train_metric>
train_tokenized = tk.texts_to_sequences(train['question_text'].fillna('missing'))
test_tokenized = tk.texts_to_sequences(test['question_text'].fillna('missing'))
clf = RandomForestClassifier(n_estimators=300, max_depth=6,max_features=11,criterion="gini",n_jobs=-1, random_state=42)
 clf.fit(x_train, y_train)
 y_predicted = clf.predict(x_val)
 accuracy, precision, recall, f1 = get_metrics(y_val, y_predicted)
 print("accuracy = %.3f, precision = %.3f, recall = %.3f, f1 = %.3f" %(accuracy, precision, recall, f1))<compute_train_metric>
max_len = 72
maxlen = 72
X_train = pad_sequences(train_tokenized, maxlen = max_len)
 X_test = pad_sequences(test_tokenized, maxlen = max_len )
y_predicted = clf.predict(x_train)
 accuracy, precision, recall, f1 = get_metrics(y_train, y_predicted)
 print("accuracy = %.3f, precision = %.3f, recall = %.3f, f1 = %.3f" %(accuracy, precision, recall, f1))<train_model>
y_train = train['target'].values
clf = XGBClassifier(n_estimators= 300, learning_rate=0.3, max_depth=4)
 clf.fit(x_train, y_train)
 y_predicted = clf.predict(x_val)
 accuracy, precision, recall, f1 = get_metrics(y_val, y_predicted)
 print("accuracy = %.3f, precision = %.3f, recall = %.3f, f1 = %.3f" %(accuracy, precision, recall, f1))<compute_train_metric>
def sigmoid(x):
return 1 /(1 + np.exp(-x))
y_predicted = clf.predict(x_train)
 accuracy, precision, recall, f1 = get_metrics(y_train, y_predicted)
 print("train accuracy = %.3f, precision = %.3f, recall = %.3f, f1 = %.3f" %(accuracy, precision, recall, f1))<train_model>
splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=10 ).split(X_train, y_train))
clf = LogisticRegression(C=2.0, solver="newton-cg", penalty="l2", n_jobs=-1)
 clf.fit(x_train, y_train)
 y_predicted = clf.predict(x_val)
 accuracy, precision, recall, f1 = get_metrics(y_val, y_predicted)
 print("accuracy = %.3f, precision = %.3f, recall = %.3f, f1 = %.3f" %(accuracy, precision, recall, f1))<compute_train_metric>
embed_size = 300
embedding_path = ".. /input/embeddings/glove.840B.300d/glove.840B.300d.txt"
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embedding_index = dict(get_coefs(*o.split(" ")) for o in open(embedding_path, encoding='utf-8', errors='ignore'))
emb_mean,emb_std = -0.005838499, 0.48782197
word_index = tk.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words + 1, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embedding_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
y_predicted = clf.predict(x_train)
 accuracy, precision, recall, f1 = get_metrics(y_train, y_predicted)
 print("train accuracy = %.3f, precision = %.3f, recall = %.3f, f1 = %.3f" %(accuracy, precision, recall, f1))<compute_train_metric>
embedding_path = ".. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt"
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embedding_index = dict(get_coefs(*o.split(" ")) for o in open(embedding_path, encoding='utf-8', errors='ignore')if len(o)>100)
 emb_mean,emb_std = -0.0053247833, 0.49346462
embedding_matrix1 = np.random.normal(emb_mean, emb_std,(nb_words + 1, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embedding_index.get(word)
 if embedding_vector is not None: embedding_matrix1[i] = embedding_vector
clf = GaussianNB()
clf.fit(x_train, y_train)
 y_predicted = clf.predict(x_val)
 accuracy, precision, recall, f1 = get_metrics(y_val, y_predicted)
 print("accuracy = %.3f, precision = %.3f, recall = %.3f, f1 = %.3f" %(accuracy, precision, recall, f1))<compute_train_metric>
embedding_matrix = np.mean([embedding_matrix, embedding_matrix1], axis=0)
 del embedding_matrix1
y_predicted = clf.predict(x_train)
 accuracy, precision, recall, f1 = get_metrics(y_train, y_predicted)
 print("train accuracy = %.3f, precision = %.3f, recall = %.3f, f1 = %.3f" %(accuracy, precision, recall, f1))<compute_train_metric>
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1)
 class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self ).__init__()
hidden_size = 128
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.1)
 self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm_attention = Attention(hidden_size*2, maxlen)
 self.gru_attention = Attention(hidden_size*2, maxlen)
 self.linear = nn.Linear(1024, 16)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.out = nn.Linear(16, 1)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 h_lstm_atten = self.lstm_attention(h_lstm)
 h_gru_atten = self.gru_attention(h_gru)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 conc = torch.cat(( h_lstm_atten, h_gru_atten, avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out
clf = BernoulliNB(alpha=0.2)
 clf.fit(x_train, y_train)
 y_predicted = clf.predict(x_val)
 accuracy, precision, recall, f1 = get_metrics(y_val, y_predicted)
 print("accuracy = %.3f, precision = %.3f, recall = %.3f, f1 = %.3f" %(accuracy, precision, recall, f1))<compute_train_metric>
m = NeuralNet()
y_predicted = clf.predict(x_train)
 accuracy, precision, recall, f1 = get_metrics(y_train, y_predicted)
 print("train accuracy = %.3f, precision = %.3f, recall = %.3f, f1 = %.3f" %(accuracy, precision, recall, f1))<compute_train_metric>
def train_model(model, x_train, y_train, x_val, y_val, validate=True):
optimizer = torch.optim.Adam(model.parameters())
 train = torch.utils.data.TensorDataset(x_train, y_train)
 valid = torch.utils.data.TensorDataset(x_val, y_val)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 loss_fn = torch.nn.BCEWithLogitsLoss(reduction='mean' ).cuda()
best_score = -np.inf
for epoch in range(n_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for x_batch, y_batch in tqdm(train_loader, disable=True):
y_pred = model(x_batch)
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds = np.zeros(( x_val_fold.size(0)))
 if validate:
avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
search_result = threshold_search(y_val.cpu().numpy() , valid_preds)
 val_f1, val_threshold = search_result['f1'], search_result['threshold']
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t val_f1={:.4f} best_t={:.2f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, avg_val_loss, val_f1, val_threshold, elapsed_time))
else:
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, elapsed_time))
valid_preds = np.zeros(( x_val_fold.size(0)))
 avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
print('Validation loss: ', avg_val_loss)
 test_preds = np.zeros(( len(test_loader.dataset)))
 for i,(x_batch,)in enumerate(test_loader):
y_pred = model(x_batch ).detach()
test_preds[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
return valid_preds, test_preds
clf = SVC(C=40)
 clf.fit(x_train, y_train)
 y_predicted = clf.predict(x_val)
 accuracy, precision, recall, f1 = get_metrics(y_val, y_predicted)
 print("accuracy = %.3f, precision = %.3f, recall = %.3f, f1 = %.3f" %(accuracy, precision, recall, f1))<compute_train_metric>
x_test_cuda = torch.tensor(X_test, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 batch_size = 512
test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False )
y_predicted = clf.predict(x_train)
 accuracy, precision, recall, f1 = get_metrics(y_train, y_predicted)
 print("train accuracy = %.3f, precision = %.3f, recall = %.3f, f1 = %.3f" %(accuracy, precision, recall, f1))<compute_train_metric>
seed=1029
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)], disable=True):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()
clf = RandomForestClassifier(n_estimators=100, max_depth=6,max_features=11, min_samples_leaf=0.0001,criterion="gini",n_jobs=-1, random_state=42)
 accuracies = cross_val_score(clf, train_encoded, y_data, cv=5, scoring="accuracy")
 print("CV accuracy", accuracies.mean() )<compute_train_metric>
train_preds = np.zeros(len(train))
test_preds = np.zeros(( len(test), len(splits)))
 n_epochs = 5
for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(X_train[train_idx], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(y_train[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(X_train[valid_idx], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(y_train[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()
train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 seed_everything(seed + i)
 model = NeuralNet()
model.cuda()
valid_preds_fold, test_preds_fold = train_model(model,
x_train_fold,
y_train_fold,
x_val_fold,
y_val_fold, validate=False)
 train_preds[valid_idx] = valid_preds_fold
test_preds[:, i] = test_preds_fold
<load_from_csv><EOS> 
search_result = threshold_search(y_train, train_preds)
 sub['prediction'] = test_preds.mean(1)> search_result['threshold']
sub.to_csv("submission.csv", index=False )