Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<create_dataframe>
%reload_ext autoreload
%autoreload 2
Results = pd.DataFrame({'Model': [],'Accuracy': [], 'Recall':[],
'Precision':[], 'F1_score':[],
'Roc_Auc':[], 'Log_loss':[],
'Positive Samples':[],
})
 def model_evaluators(y_valid, preds, preds_proba, modelName):
tn, fp, fn, tp = confusion_matrix(y_valid, preds ).ravel()
acc_ =(tp + tn)/(tp + tn + fn + fp)
 sens_ = tp /(tp + fn)
 sp_ = tn /(tn + fp)
 PPV = tp /(tp + fp)
 NPV = tn /(tn + fn)
 F1 = 2*(PPV*sens_)/(PPV+sens_)
 fpr, tpr, thr = metrics.roc_curve(y_valid, preds_proba)
 rocauc = auc(fpr, tpr)
 log_loss_ = log_loss(y_valid, preds_proba)
 correct_guesses = accuracy_score(y_valid, preds, normalize=False)
 res = pd.DataFrame({'Model': [modelName],
'Accuracy': ["%.3f"% acc_],
'Recall':["%.3f"% sens_],
'Precision':["%.3f"% PPV],
'F1_score':["%.3f"% F1],
'Roc_Auc':["%.3f"% rocauc],
'Log_loss':["%.3f"% log_loss_],
'Positive Samples':[correct_guesses]
})
 return res<predict_on_test>
import gc
import os
import pickle
import random
import re
import sklearn.metrics
import sklearn.model_selection
import time
import warnings
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from collections import OrderedDict
from tqdm import tqdm, tqdm_notebook
import functools
import multiprocessing
import sklearn.preprocessing
import unicodedata
import copy
import time
from sklearn.model_selection import train_test_split, StratifiedKFold
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.data
predictors = [i for i in X_train.columns if i not in ['PassengerId']]
def modelfit_eval(clf, X_train, y_train, X_valid, y_valid, predictors, clf_name):
model = clf
model.fit(X_train[predictors], y_train)
 preds = model.predict(X_valid[predictors])
 preds_proba = model.predict_proba(X_valid[predictors])[:, 1]
model_xgb = model
res = model_evaluators(y_valid, preds, preds_proba, clf_name)
 return model, res
model_xgb, res = modelfit_eval(XGBClassifier() , X_train, y_train, X_valid, y_valid, predictors, 'XGB Classifier')
 Results = Results.append(res, ignore_index=True)
 model_log, res = modelfit_eval(LogisticRegression(solver='liblinear'), X_train, y_train, X_valid, y_valid, predictors, 'Logistic Regression')
 Results = Results.append(res, ignore_index=True)
 model_dtree, res = modelfit_eval(DecisionTreeClassifier() , X_train, y_train, X_valid, y_valid, predictors, 'Decision Tree Classifier')
 Results = Results.append(res, ignore_index=True)
 model_rf, res = modelfit_eval(RandomForestClassifier() , X_train, y_train, X_valid, y_valid, predictors, 'Random Forest Classifier')
 Results = Results.append(res, ignore_index=True)
 model_svm, res = modelfit_eval(SVC(gamma=0.001, probability=True), X_train, y_train, X_valid, y_valid, predictors, 'SVC')
 Results = Results.append(res, ignore_index=True)
 model_nb, res = modelfit_eval(GaussianNB() , X_train, y_train, X_valid, y_valid, predictors, 'Gaussian Naive Bayes')
 Results = Results.append(res, ignore_index=True)
 Results<train_on_grid>
def tf_seed_everything(seed):
os.environ['PYTHONHASHSEED'] = str(seed)
 random.seed(seed)
 np.random.seed(seed)
 tf.set_random_seed(seed )
def modelfit(model, X_train, y_train, predictors, useTrainCV=True, cv_folds=5, early_stopping_rounds=50):
if useTrainCV:
xgb_param = model.get_xgb_params()
xgtrain = xgb.DMatrix(X_train[predictors].values,
label=y_train.values)
 cvresult = xgb.cv(xgb_param,
xgtrain,
num_boost_round=model.get_params() ['n_estimators'],
nfold=cv_folds,
metrics='auc',
early_stopping_rounds=early_stopping_rounds)
 model.set_params(n_estimators=cvresult.shape[0])
 print("Best iteration:", cvresult.shape[0])
 print(cvresult.tail(1))
model.fit(X_train[predictors], y_train, eval_metric='auc')
 predictions = model.predict(X_train[predictors])
 predprob = model.predict_proba(X_train[predictors])[:,1]
print("
Model Report")
 print("Accuracy : %.4g" % metrics.accuracy_score(y_train.values, predictions))
print("AUC Score(Train): %f" % metrics.roc_auc_score(y_train, predprob))
feat_imp = pd.Series(model.get_booster().get_fscore() ).sort_values(ascending=False)
 feat_imp.plot(kind='bar', title='Feature Importances')
 plt.ylabel('Feature Importance Score' )<choose_model_class>
def f1_curve(target, preds, t_min=0.01, t_max=0.99, steps=99):
curve = {}
for t in np.linspace(t_min, t_max, steps):
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 curve[t] = sklearn.metrics.f1_score(target, preds >= t)
 return pd.Series(curve ).sort_index()
xgb1 = XGBClassifier(learning_rate =0.1,
n_estimators=1000,
max_depth=4,
min_child_weight=0,
gamma=0,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27)
 modelfit(xgb1, X_train, y_train, predictors )<choose_model_class>
class QuoraData:
def __init__(self):
self.paths = {
'glove': '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt',
'news': '.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin',
'paragram': '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt',
'wiki': '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec',
}
def glove(self): return QuoraEmbedding(self.paths['glove'])
 def news(self): return QuoraEmbedding(self.paths['news'])
 def paragram(self): return QuoraEmbedding(self.paths['paragram'])
 def wiki(self): return QuoraEmbedding(self.paths['wiki'])
 def convert_start(self, embeddings):

 self.convert_pids = []
for name in embeddings:
path = self.paths[name]
if path.endswith('.npy'): continue
if os.path.exists(path + '.npy'): continue
if os.path.exists(name + '.npy'): continue
if KAGGLE_RUN:
out_path = name + '.npy'
else:
out_path = path + '.npy'
print(f'Converting {path} -> {out_path}', flush=True)
 pid = os.fork()
if pid == 0:
emb = QuoraEmbedding(path)
 emb.save_npy(out_path)
 os._exit(0)
 self.convert_pids.append(pid)
 self.paths[name] = out_path
def convert_wait(self):

 for pid in self.convert_pids:
try:
os.waitpid(pid, 0)
 except:
pass
def read_train(self):
return pd.read_csv('.. /input/train.csv')
 def read_test(self):
return pd.read_csv('.. /input/test.csv')
 def read_input(self):
return pd.concat([self.read_train() , self.read_test() ], axis=0, copy=False, sort=False, ignore_index=True )
param_test1 = {'max_depth':range(3,10,1),
'min_child_weight':range(0,6,1)}
gsearch1 = GridSearchCV(estimator = XGBClassifier(learning_rate =0.1,
n_estimators=3,
max_depth=4,
min_child_weight=0,
gamma=0,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27),
param_grid = param_test1,
scoring='roc_auc',
n_jobs=4,
cv=5)
 gsearch1.fit(X_train[predictors],y_train)
 def grid_scores(model):
print("Best parameters set found on development set:")
 print(model.best_params_)
 print("
Grid scores on development set:")
 means = model.cv_results_['mean_test_score']
stds = model.cv_results_['std_test_score']
for mean, std, params in zip(means, stds, model.cv_results_['params']):
print("%0.3f(+/-%0.03f)for %r"
%(mean, std * 2, params))
grid_scores(gsearch1 )<choose_model_class>
class QuoraEmbedding:

 def __init__(self, filename, vectors=None, vocab=None):
self.filename = filename
if vectors is None:
vectors, vocab = self._read(filename)
 self.vectors = vectors
self.index2word = vocab
self.name = re.findall('^\w+', os.path.basename(filename)) [0]
self.shape = self.vectors.shape
self.num_words, self.dim = self.shape
assert len(self.index2word)== self.num_words
self.word2index = { self.index2word[i]: i for i in reversed(range(self.num_words)) }
self.lword2index = { self.index2word[i].lower() : i for i in reversed(range(self.num_words)) }
def __repr__(self):
return f'QuoraEmbedding({self.name}, {self.shape[0]}x{self.shape[1]}, {self.vectors.nbytes/(1024**3):.1f}GiB)'
def lookup(self, word, lower=False):
if lower:
idx = self.lword2index.get(word.lower() , -1)
 else:
idx = self.word2index.get(word, -1)
 return self.vectors[idx] if idx != -1 else None
@functools.lru_cache(1)
 def mean(self): return self.vectors.mean()
@functools.lru_cache(1)
 def std(self): return self.vectors.std()
def save_npy(self, filename):
assert filename.endswith('.npy')
 np.save(filename, self.vectors)
 with open(filename[:-4] + '.vocab', 'wb')as fp:
pickle.dump(self.index2word, fp)
 @classmethod
def _read(cls, filename):

 if filename.endswith('.npy'):
return cls._read_npy(filename)
 elif os.path.exists(filename + '.npy'):
return cls._read_npy(filename + '.npy')
 elif os.path.exists(re.sub('[.]\w+$', '.npy', filename)) :
return cls._read_npy(re.sub('[.]\w+$', '.npy', filename))
elif filename.endswith('.bin'):
return cls._read_bin(filename)
 else:
return cls._read_txt(filename)
 @classmethod
def _read_npy(cls, filename):
assert filename.endswith('.npy')
 vectors = np.load(filename, 'r')
 with open(filename[:-4] + '.vocab', 'rb')as fp:
vocab = pickle.load(fp)
 return vectors, vocab
@classmethod
def _read_txt(cls, filename):
vectors = []
vocab = []
dim = -1
for line_num, line in enumerate(open(filename, 'rb')) :
try:
line = line.decode('utf-8')
 except:
line = line.decode('latin_1')
 print(f'Bad word on line {line_num+1}: {line[:20]}...')
 word, line = line.split(' ', maxsplit=1)
 vec = np.fromstring(line, np.float32, count=dim, sep=' ')
 if len(vec)== 1 and len(vectors)== 0:
dim = int(vec[0])
 continue
if dim != -1 and dim != len(vec):
raise Exception(f'Mismatching vector lengths: {dim} vs {len(vec)}')
 dim = len(vec)
 vectors.append(vec)
 vocab.append(word)
 vectors = np.stack(vectors)
 vocab = '
'.join(vocab ).split('
')
 return vectors, vocab
@classmethod
def _read_bin(cls, filename):
kv = KeyedVectors.load_word2vec_format(filename, binary=True)
 return kv.vectors, kv.index2word
param_test3 = {'gamma':[i/10.0 for i in range(0,10)]}
gsearch3 = GridSearchCV(estimator = XGBClassifier(learning_rate =0.1,
n_estimators=3,
max_depth=4,
min_child_weight=1,
gamma=0,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27),
param_grid = param_test3,
scoring='roc_auc',
n_jobs=4,
cv=5)
 gsearch3.fit(X_train[predictors],y_train)
 grid_scores(gsearch3 )<choose_model_class>
class QuoraFeatureExtractor:
def __init__(self, num_words=120000, max_len=70, want_capstate=False, want_wide=False, preprocessor=None):
self.max_len = max_len
self.num_words = num_words
self.num_aux = 1
self.preprocessor = preprocessor
self.want_capstate = want_capstate
self.want_wide = want_wide
def fit_transform(self, texts, fit_mask=None):
if fit_mask is None:
fit_mask = np.full(len(texts), True, dtype='bool')
 if self.preprocessor:
print('Preprocessing', flush=True)
 texts = self.preprocessor.transform(texts)
 fit_mask = np.array(fit_mask, dtype='bool')
 print('Tokenizing', flush=True)
 self.tokenizer = Tokenizer(num_words=self.num_words, filters='', split=' ', oov_token='__')
 self.tokenizer.fit_on_texts(texts[fit_mask])
 self.word_index = self.tokenizer.word_index
tokens = self.tokenizer.texts_to_sequences(texts)
 tokens = pad_sequences(tokens, maxlen=self.max_len)
 res = {'tokens': tokens}
if self.want_capstate:
print('Generating capstate', flush=True)
 with multiprocessing.Pool(multiprocessing.cpu_count())as pool:
capstate = np.stack(pool.imap(self.map_capstate, texts, chunksize=10000))
res['capstate'] = capstate
if self.want_wide:
print('Generating wide features', flush=True)
 wide = self.gen_wide_features(texts)
 self.ss = sklearn.preprocessing.RobustScaler()
self.ss.fit(wide[fit_mask].values)
 wide = self.ss.transform(wide.values)
 wide[wide > 3] = 3
res['wide'] = wide
return res
def map_capstate(self, text):
toks = text.split(' ')
 res = np.zeros(self.max_len, dtype=np.int8)
 for i in range(min(len(res), len(toks))):
tok = toks[len(toks)- 1 - i]
x = 0
if tok[0].isdigit() :
x = 6
elif unicodedata.category(tok[0] ).startswith('P'):
x = 5
elif tok == tok.lower() :
x = 1
elif tok == tok.capitalize() :
x = 2
elif tok == tok.upper() :
x = 3
else:
x = 4
res[-i-1] = x
return res
def gen_wide_features(self, texts):
texts = pd.Series(texts)
 wide = pd.DataFrame()
wide['num_chars'] = texts.str.len()
wide['num_caps'] = texts.str.count('[A-Z]')
 wide['frac_caps'] =(wide.num_caps / wide.num_chars ).fillna(0)
 wide['num_dots'] = texts.str.count('[.]')
 return wide
def embedding_weights(self, emb, verbose=1):

 W = np.random.normal(emb.mean() , emb.std() ,(self.num_words, emb.dim))
miss = 0
for word, idx in self.word_index.items() :
if idx >= self.num_words: continue
vec = emb.lookup(word, lower=True)
 if vec is not None:
W[idx] = vec
else:
miss += 1
if verbose > 0:
print(f'{miss} words from tokenizer missing({miss/self.num_words:.2f}%)in {emb.name}')
 return W
xgb2 = XGBClassifier(learning_rate =0.1,
n_estimators=1000,
max_depth=4,
min_child_weight=1,
gamma=0.6,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27)
 modelfit(xgb2, X_train, y_train, predictors )<choose_model_class>
def torch_seed(seed):
os.environ['PYTHONHASHSEED'] = str(seed)
 random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
param_test4 = {'subsample':[i/10.0 for i in range(6,10)],
'colsample_bytree':[i/10.0 for i in range(6,10)]}
gsearch4 = GridSearchCV(estimator = XGBClassifier(learning_rate =0.1,
n_estimators=8,
max_depth=4,
min_child_weight=1,
gamma=0.6,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27),
param_grid = param_test4,
scoring='roc_auc',
n_jobs=4,
cv=5)
 gsearch4.fit(X_train[predictors], y_train)
 grid_scores(gsearch4 )<choose_model_class>
def torch_gc() :
gc.collect()
torch.cuda.empty_cache()
param_test5 = {'subsample':[i/100.0 for i in range(85,100,5)],
'colsample_bytree':[i/100.0 for i in range(75,90,5)]}
gsearch5 = GridSearchCV(estimator = XGBClassifier(learning_rate =0.1,
n_estimators=8,
max_depth=4,
min_child_weight=1,
gamma=0.6,
subsample=0.9,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27),
param_grid = param_test5,
scoring='roc_auc',
n_jobs=4,
cv=5)
 gsearch5.fit(X_train[predictors],y_train)
 grid_scores(gsearch5 )<define_search_space>
def keras_rnn_init(m):
for name, param in m.named_parameters() :
if 'weight_ih' in name: torch.nn.init.xavier_uniform_(param)
 if 'weight_hh' in name: torch.nn.init.orthogonal_(param)
 if 'bias_' in name: torch.nn.init.constant_(param, 0 )
param_test6 = {'reg_alpha':[1e-5, 0.0005, 1e-2, 0.1, 0.5, 1, 100]}
gsearch6 = GridSearchCV(estimator = XGBClassifier(learning_rate =0.1,
n_estimators=8,
max_depth=4,
min_child_weight=1,
gamma=0.6,
subsample=0.9,
colsample_bytree=0.75,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27),
param_grid = param_test6,
scoring='roc_auc',
n_jobs=4,
cv=5)
 gsearch6.fit(X_train[predictors],y_train)
 grid_scores(gsearch6 )<choose_model_class>
class Fold:
def __init__(self, data, X, seed=42, valid_frac=0.1, holdout_frac=0.0432, holdout_seed=42):
self.data = data
self.X = X
self.y = data.target.values.astype(np.float32)
 self.seed = seed
tmask = data.target.notnull().values
self.all_idx = data.index.values.astype(np.int32)
 self.test_idx = self.all_idx[~tmask]
trainval_idx, self.holdout_idx = train_test_split(
 self.all_idx[tmask],
stratify=self.y[tmask],
test_size=holdout_frac,
random_state=holdout_seed,
shuffle=True)
 self.train_idx, self.valid_idx = train_test_split(
 trainval_idx,
stratify=self.y[trainval_idx],
test_size=valid_frac,
random_state=seed,
shuffle=True)
 self.train_idx.sort()
self.valid_idx.sort()
self.holdout_idx.sort()
self.oob_idx = self.all_idx[~np.isin(self.all_idx, self.train_idx, assume_unique=True)]
param_test7 = {'reg_alpha':[1e-07, 1e-06, 1e-05, 1e-04, 1e-03, 1e-02]}
gsearch7 = GridSearchCV(estimator = XGBClassifier(learning_rate =0.1,
n_estimators=8,
max_depth=4,
min_child_weight=1,
gamma=0.6,
subsample=0.9,
colsample_bytree=0.75,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27),
param_grid = param_test7,
scoring='roc_auc',
n_jobs=4,
cv=5)
 gsearch7.fit(X_train[predictors],y_train)
 grid_scores(gsearch7 )<choose_model_class>
%%time
torch_seed(42)
 qd = QuoraData()
qd.convert_start(['glove', 'wiki'])
 prep = QuoraPreprocessor()
input_df = qd.read_input()
input_df['question_text'] = prep.transform(input_df.question_text )
xgb3 = XGBClassifier(learning_rate =0.1,
n_estimators=8,
max_depth=4,
min_child_weight=1,
gamma=0.6,
subsample=0.9,
colsample_bytree=0.75,
reg_alpha=1e-07,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27)
 modelfit(xgb3, X_train, y_train, predictors )<choose_model_class>
%%time
qfe = QuoraFeatureExtractor(num_words=95000, max_len=70)
 input_X = qfe.fit_transform(input_df.question_text, fit_mask=input_df.target.notnull().values)
 print({ k: v.shape for(k, v)in input_X.items() } )
xgb4 = XGBClassifier(learning_rate =0.001,
n_estimators=200,
max_depth=4,
min_child_weight=1,
gamma=0.6,
subsample=0.9,
colsample_bytree=0.75,
reg_alpha=1e-07,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27)
 modelfit(xgb4, X_train, y_train, predictors )<predict_on_test>
%%time
qd.convert_wait()
torch_seed(42)
 emb_glove = qfe.embedding_weights(qd.glove())
 torch_seed(43)
 emb_wiki = qfe.embedding_weights(qd.wiki())
 emb_glovewiki = np.concatenate([emb_glove, emb_wiki], axis=1)
 gc.collect()
os.system('rm -f glove.npy glove.vocab wiki.npy wiki.vocab' )
model_xgb = xgb4
model_xgb.fit(X_train[predictors], y_train)
 preds = model_xgb.predict(X_valid[predictors])
 preds_proba = model_xgb.predict_proba(X_valid[predictors])[:, 1]
res = model_evaluators(y_valid, preds, preds_proba, 'Hyperparametarized XGB')
 Results = Results.append(res, ignore_index=True)
 Results<define_search_space>
class Arch1b_LstmGru(torch.nn.Module):
def __init__(self, emb_weights):
super().__init__()
self.embedding = nn.Embedding(emb_weights.shape[0], emb_weights.shape[1])
 self.embedding.weight = nn.Parameter(torch.Tensor(emb_weights), requires_grad=False)
 self.lstm = nn.LSTM(self.embedding.embedding_dim, 128, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(self.lstm.hidden_size*2, 64, bidirectional=True, batch_first=True)
 self.fc = nn.Linear(128, 1)
 def dropout1d(self, x, p):
x = x.permute(0, 2, 1)
 x = F.dropout2d(x, p, training=self.training)
 x = x.permute(0, 2, 1)
 return x
def forward(self, x_tok):
x = self.embedding(x_tok)
 x = self.dropout1d(x, 0.2)
 x = F.dropout2d(x, 0.05, training=self.training)
 x, _ = self.lstm(x)
 x, c = self.gru(x)
 x = x.max(1)[0]
x = F.dropout(x, 0.1, training=self.training)
 x = self.fc(x)
 return x
rf_grid_param = {'n_estimators': [100, 200, 300, 400],
'max_features': ['auto', 'sqrt', 'log2'],
'max_depth': [4, 5, 7, None],
'min_samples_split': [2, 3, 5, 7],
'min_samples_leaf': [1, 3, 5, 7]}
dt_grid_param = {'criterion': ['gini', 'entropy'],
'max_features': ['auto', 'sqrt', 'log2'],
'max_depth': [4, 5, 7, None],
'min_samples_split': [2, 3, 5, 7],
'min_samples_leaf': [1, 3, 5, 7]}
log_grid_param = {'penalty': ['l1', 'l2'],
'C': np.logspace(-4, 4, 20),
'solver': ['liblinear', 'saga']}
nb_grid_param = {'var_smoothing': np.logspace(0,-9, num=100)}
model_rf = RandomizedSearchCV(estimator=model_rf,
param_distributions=rf_grid_param,
n_iter=300,
cv =5,
scoring='roc_auc',
verbose=2,
random_state=27,
n_jobs = -1)
 model_dtree = RandomizedSearchCV(estimator=model_dtree,
param_distributions=dt_grid_param,
n_iter=300,
cv =5,
scoring='roc_auc',
verbose=1,
random_state=27,
n_jobs = -1)
 model_log = RandomizedSearchCV(estimator=model_log,
param_distributions =log_grid_param,
n_iter=300,
cv = 5,
scoring='roc_auc',
verbose=1,
random_state=27,
n_jobs=-1)
 model_nb = RandomizedSearchCV(estimator=model_nb,
param_distributions=nb_grid_param,
n_iter=300,
cv = 5,
scoring='roc_auc',
verbose=1,
random_state=27,
n_jobs=-1)
 preds_list = []
for model in [model_rf, model_dtree, model_log, model_nb]:
model.fit(X_train[predictors], y_train)
 model = model.best_estimator_
preds = model.predict(X_valid[predictors])
 preds_proba = model.predict_proba(X_valid[predictors])[:, 1]
clf_name = str(model ).split('(')[0]
res = model_evaluators(y_valid, preds, preds_proba, 'Hyperparametarized '+clf_name)
 Results = Results.append(res, ignore_index=True)
 preds_list.append(preds )<predict_on_test>
class Learner:
def __init__(self, fold, model, seed=None, batch_size=512, grad_clip=1):
self.__dict__.update(fold.__dict__)
 self.fold = fold
self.model = model
self.dataset = torch.utils.data.TensorDataset(
 torch.LongTensor(self.X['tokens']),
torch.FloatTensor(self.y[:, np.newaxis]))
self.train_dl = self.make_loader(self.train_idx, batch_size=batch_size, shuffle=True)
 self.valid_dl = self.make_loader(self.valid_idx)
 opt_params = [p for p in self.model.parameters() if p.requires_grad]
self.opt = torch.optim.Adam(opt_params)
 self.grad_clip = grad_clip
self.loss_fn = nn.BCEWithLogitsLoss(reduction='sum')
 self.steps = 0
self.epochs = 0
params_sz = sum([p.detach().cpu().numpy().nbytes for p in opt_params])
 print(f'Training on {len(self.train_dl.dataset)} examples, '
f'validating on {len(self.valid_dl.dataset)} examples.'
f'Parameters: {params_sz/1048576:.2f}MiB', flush=True)
 self.epoch_params = [ self.get_params() ]
self.val_metrics = []
def make_loader(self, idx, batch_size=1024, shuffle=False):
ds = torch.utils.data.Subset(self.dataset, idx)
 return torch.utils.data.DataLoader(
 ds, batch_size=batch_size, shuffle=shuffle, pin_memory=True)
 def get_params(self):
return { name: param.detach().cpu().numpy().copy()
for(name, param)in self.model.named_parameters()
if param.requires_grad }
def restore_params(self, prev):
for name, param in self.model.named_parameters() :
if name in prev:
param.data.copy_(torch.from_numpy(prev[name]))
def restore_epoch(self, epoch):
self.restore_params(self.epoch_params[epoch])
 def predict(self, idx=None, dl=None):
if dl is None:
dl = self.make_loader(idx, shuffle=False)
 preds = []
with torch.no_grad() :
self.model.eval()
for batch in dl:
batch = [t.cuda() for t in batch[:-1]]
pred = torch.sigmoid(self.model(*batch))
preds.append(pred.cpu().numpy().flatten())
 self.model.train()
return np.concatenate(preds)
 def eval(self):
y_pred = self.predict(dl=self.valid_dl)
 y_true = self.y[self.valid_idx]
curve = f1_curve(y_true, y_pred)
 metrics = {
'epochs': self.epochs,
'steps': self.steps,
'f1': curve.max() ,
'roc-auc': sklearn.metrics.roc_auc_score(y_true, y_pred),
'pr-auc': sklearn.metrics.average_precision_score(y_true, y_pred),
'loss': sklearn.metrics.log_loss(y_true, y_pred),
'thresh': curve.idxmax() ,
}
self.val_metrics.append(metrics)
 print(' '.join('%s=%.6g' %(k, float(v)) for(k, v)in metrics.items()), flush=True)
 def train(self, epochs=1, eval_steps=0):
self.model.cuda()
self.model.train()
for ep in range(epochs):
t_start = time.time()
if not KAGGLE_RUN and eval_steps > 0:
it = enumerate(tqdm_notebook(self.train_dl, leave=False))
else:
it = enumerate(self.train_dl)
 for i, batch in it:
batch = [t.cuda() for t in batch]
y_batch = batch.pop()
self.opt.zero_grad()
y_pred = self.model(*batch)
 loss = self.loss_fn(y_pred, y_batch)
 loss.backward()
if self.grad_clip is not None:
nn.utils.clip_grad_norm_(self.model.parameters() , self.grad_clip)
 self.opt.step()
self.steps += 1
if eval_steps and i % eval_steps == 0 and i > 0: self.eval()
self.epoch_params.append(self.get_params())
 self.epochs += 1
print('t=%.2fs' %(time.time() - t_start), end=' ', flush=True)
 self.eval()
return self
preds_proba = model_rf.best_estimator_.predict_proba(X_valid[predictors])[:, 1]
X_valid['preds_proba'] = preds_proba
X_valid['preds'] = preds_list[0]
X_valid['true_preds'] = y_valid
wrong_preds = X_valid.loc[(( X_valid.preds == 1)&(X_valid.true_preds == 0)) | 
(( X_valid.preds == 0)&(X_valid.true_preds == 1)) ].sort_values(by=['preds'],
ascending=False)
 X_valid.drop('preds_proba', axis=1, inplace=True)
 X_valid.drop('preds', axis=1, inplace=True)
 X_valid.drop('true_preds', axis=1, inplace=True )<count_values>
%%time
fold = Fold(input_df, input_X, seed=1005); torch_seed(fold.seed)
 learn1 = Learner(fold, Arch1b_LstmGru(emb_glovewiki)).train(5 )
print('Total Error examples:', wrong_preds.shape[0])
 print('-------------------------')
 print(wrong_preds['preds'].value_counts())
 print('-------------------------')
 print(wrong_preds['Sex_female'].value_counts())
 print('-------------------------')
 print(wrong_preds['Embarked'].value_counts())
 print('-------------------------')
 print(wrong_preds['Pclass'].value_counts())
 print('-------------------------')
 print(wrong_preds['Title'].value_counts())
 print('-------------------------')
 print(wrong_preds['Deck'].value_counts())
 print('-------------------------')
 print(wrong_preds['FamilySize'].value_counts() )<filter>
%%time
fold = Fold(input_df, input_X, seed=555); torch_seed(fold.seed)
 learn2 = Learner(fold, Arch1b_LstmGru(emb_glovewiki)).train(5 )
print('Unknown Decks and Lonely Travellers:',
wrong_preds.loc[(( wrong_preds.Deck < 0.522068)&(wrong_preds.Deck > 0.522066)) & 
(( wrong_preds.FamilySize < -0.560974)&(wrong_preds.FamilySize > -0.560976)) ].shape[0])
 print('Unknown Decks Lonely Travellers embarked from Southampton:',
wrong_preds.loc[(( wrong_preds.Deck < 0.522068)&(wrong_preds.Deck > 0.522066)) & 
(wrong_preds.Embarked == 2)& 
(( wrong_preds.FamilySize < -0.560974)&(wrong_preds.FamilySize > -0.560976)) ].shape[0])
 print('Mr.title and Lonely Traveller:',
wrong_preds.loc[(( wrong_preds.Title < 0.214012)&(wrong_preds.Title > 0.214010)) & 
(( wrong_preds.FamilySize < -0.560974)&(wrong_preds.FamilySize > -0.560976)) ].shape[0])
 print('Unknown Deck and Embarked from S:',
wrong_preds.loc[(( wrong_preds.Deck < 0.522068)&(wrong_preds.Deck > 0.522066)) & 
(wrong_preds.Embarked == 2)].shape[0] )<feature_engineering>
%%time
fold = Fold(input_df, input_X, seed=1111); torch_seed(fold.seed)
 learn3 = Learner(fold, Arch1b_LstmGru(emb_glovewiki)).train(5 )
def f(row):
if(row['Title'] > 0.214010 and row['Title'] < 0.214012)and(row['FamilySize'] > -0.560976 and row['FamilySize'] < -0.560974): val = 1
else: val = 0
return val
X['Mr_lonely'] = X.apply(f, axis=1)
 X_test_full['Mr_lonely'] = X_test_full.apply(f, axis=1)
 def f(row):
if(row['Deck'] > 0.522066 and row['Deck'] < 0.522068)and row['Embarked'] == 2: val = 1
else: val = 0
return val
X['Unknown_s'] = X.apply(f, axis=1)
 X_test_full['Unknown_s'] = X_test_full.apply(f, axis=1 )<split>
%%time
fold = Fold(input_df, input_X, seed=1111); torch_seed(fold.seed)
 learn3 = Learner(fold, Arch1b_LstmGru(emb_glovewiki)).train(5 )
X_train, X_valid, y_train, y_valid = train_test_split(X, y, train_size=0.7, test_size=0.3, random_state=27)
 rf_grid_param = {'n_estimators': [200, 300, 400, 700],
'max_features': ['auto', 'sqrt', 'log2'],
'max_depth': [4, 5, 7, None],
'min_samples_split': [2, 3, 5, 7],
'min_samples_leaf': [1, 3, 5, 7]}
model_rf, _ = modelfit_eval(RandomForestClassifier() , X_train, y_train, X_valid, y_valid, predictors, 'Random Forest Classifier')
 model_rf = RandomizedSearchCV(estimator=model_rf,
param_distributions=rf_grid_param,
n_iter=300,
cv =5,
scoring='roc_auc',
verbose=2,
random_state=27,
n_jobs = -1)
 model_rf.fit(X_train[predictors], y_train)
 model = model_rf.best_estimator_
preds = model.predict(X_valid[predictors])
 preds_proba = model.predict_proba(X_valid[predictors])[:, 1]
res = model_evaluators(y_valid, preds, preds_proba, 'Random Forest after Error Analysis')
 Results = Results.append(res, ignore_index=True )<save_to_csv>
%%time
fold = Fold(input_df, input_X, seed=3333); torch_seed(fold.seed)
 learn4 = Learner(fold, Arch1b_LstmGru(emb_glovewiki)).train(5 )
preds_test = model.predict(X_test_full[predictors])
 output = pd.DataFrame({'PassengerId': X_test_full.PassengerId, 'Survived': preds_test})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )<import_modules>
%%time
test_idx = fold.test_idx
hold_idx = fold.holdout_idx
hold_y = input_df.loc[fold.holdout_idx, 'target']
ensemble_hold = []
ensemble_test = []
for li, learn in enumerate([learn1, learn2, learn3, learn4]):
for ep in [5]:
if ep >= len(learn.epoch_params): continue
learn.restore_epoch(ep)
 ensemble_hold.append(learn.predict(idx=hold_idx))
f1 = f1_curve(hold_y, ensemble_hold[-1] ).max()
print('learn%d ep%d %.6f' %(li+1, ep, f1))
ensemble_test.append(learn.predict(idx=test_idx))
ensemble_test_s = pd.Series(np.mean(ensemble_test, axis=0), index=fold.data.loc[test_idx, 'qid'])
 score = f1_curve(hold_y, np.mean(ensemble_hold, axis=0))
thresh = score.idxmax()
print('F1score=',score.max())
 submission_df = pd.read_csv('.. /input/sample_submission.csv')
 submission_df['prediction'] =(ensemble_test_s.loc[submission_df.qid] >= thresh ).astype(int ).values
submission_df.to_csv('submission.csv', index=False )