Kaggle competition: Quora Insincere Questions Classification
predictions_nullblk = dtreeClass.predict(X_nullblk_test)
 predictions_nullblk<prepare_output>
seed = 123
predictions_nullblk=pd.DataFrame(predictions_nullblk, columns=['block_A','block_B','block_C','block_D','block_E','block_F','block_G'])
 predictions_nullblk['block']=np.where(predictions_nullblk['block_A']==1,'A','')
 predictions_nullblk['block']=np.where(predictions_nullblk['block_B']==1,'B',predictions_nullblk['block'])
 predictions_nullblk['block']=np.where(predictions_nullblk['block_C']==1,'C',predictions_nullblk['block'])
 predictions_nullblk['block']=np.where(predictions_nullblk['block_D']==1,'D',predictions_nullblk['block'])
 predictions_nullblk['block']=np.where(predictions_nullblk['block_E']==1,'E',predictions_nullblk['block'])
 predictions_nullblk['block']=np.where(predictions_nullblk['block_F']==1,'F',predictions_nullblk['block'])
 predictions_nullblk['block']=np.where(predictions_nullblk['block_G']==1,'G',predictions_nullblk['block'])
 predictions_nullblk.head()<prepare_output>
class DataLoader:
def load(self, file_names):
self.train_df = pd.read_csv(file_names['train'])
 self.test_df = pd.read_csv(file_names['test'])
 self.train_df['question_text'].fillna('_na_', inplace=True)
 self.test_df['question_text'].fillna('_na_', inplace=True)
 print('Train shape : ', self.train_df.shape)
 print('Test shape : ', self.test_df.shape )
predictions_nullblk=predictions_nullblk['block'].values<drop_column>
class Embeddings:
def __init__(self):
self.embed_len = 300
self.punctuation = set('!
def load(self, embedding_file_name):
self.embeddings_index = {}
f = open(embedding_file_name)
 for line in tqdm(f):
values = line.split(' ');
word = values[0]
coef = np.asarray(values[1:], dtype='float32')
 self.embeddings_index[word] = coef
self.embed_len = len(coef)
 f.close()
def text_to_vec(self, text, max_text_len=30):
if text[-1] in self.punctuation:
text = text[:-1].split() + [text[-1]]
else:
text = text.split()
text = text[:max_text_len]
empyt_emb = np.zeros(self.embed_len)
 embeds = [self.embeddings_index.get(word, empyt_emb)for word in text]
embeds+= [empyt_emb] *(max_text_len - len(embeds))
return np.array(embeds)
 def sequences_to_vec(self, text_sequences, max_text_len=30):
vectors = [self.text_to_vec(text, max_text_len)for text in text_sequences]
return np.array(vectors )
nullblk.drop(columns=['block'])
 nullblk['block']=predictions_nullblk
nullblk<categorify>
def batch_gen(train_df, batch_size, emb, max_text_len=30):
n_batches = math.ceil(len(train_df)/ batch_size)
 while True:
train_df = train_df.sample(frac=1.)
for i in range(n_batches):
batch_df = train_df.iloc[i*batch_size:(i+1)*batch_size]
X_ = emb.sequences_to_vec(batch_df["question_text"], max_text_len)
 y_ = batch_df["target"].values
yield X_, y_
def batch_gen_test(test_df, emb, batch_size_test=256, max_text_len=30):
n_batches = math.ceil(len(test_df)/ batch_size)
 for i in range(n_batches):
batch_df = test_df.iloc[i*batch_size:(i+1)*batch_size]
X_ = emb.sequences_to_vec(batch_df["question_text"], max_text_len)
 yield X_
df['CabinOccupancy'] = df['CabinOccupancy'].mask(df['CabinOccupancy'].eq(0)).fillna(df['PassengerId'].map(nullblk.set_index('PassengerId')['CabinOccupancy']))
df['block'] = df['block'].mask(df['block'].eq(0)).fillna(df['PassengerId'].map(nullblk.set_index('PassengerId')['block'])) 
<filter>
file_names = {'train': '.. /input/train.csv', 'test': '.. /input/test.csv'}
dl = DataLoader()
dl.load(file_names )
df[df['PassengerId']==9]<merge>
embedding_file_name = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
emb = Embeddings()
emb.load(embedding_file_name)
 print('Embedding loaded' )
train['Age'] = train['Age'].mask(train['Age'].eq(0)).fillna(train['PassengerId'].map(df.set_index('PassengerId')['Age']))
train['Embarked'] = train['Embarked'].mask(train['Embarked'].eq(0)).fillna(train['PassengerId'].map(df.set_index('PassengerId')['Embarked']))
train=pd.merge(train,df[['PassengerId','CabinOccupancy','block','Titles']],how='left',on='PassengerId' )<import_modules>
val_size = 0.03
train_df, val_df, _, _ = train_test_split(dl.train_df, dl.train_df['target'],
test_size=val_size,
stratify=dl.train_df['target'],
shuffle=True, random_state=seed)
 val_X = emb.sequences_to_vec(val_df["question_text"], max_text_len)
 val_y = val_df['target'].values
import scipy
import scipy.stats as ss
from scipy.stats import pearsonr
from scipy import stats 
<statistical_test>
from keras.models import Sequential
from keras.layers import CuDNNLSTM, Dense, Bidirectional
def cramers_v(confusion_matrix):

 chi2 = ss.chi2_contingency(confusion_matrix)[0]
n = confusion_matrix.sum()
phi2 = chi2 / n
r, k = confusion_matrix.shape
phi2corr = max(0, phi2 -(( k-1)*(r-1)) /(n-1))
rcorr = r -(( r-1)**2)/(n-1)
 kcorr = k -(( k-1)**2)/(n-1)
 return np.sqrt(phi2corr / min(( kcorr-1),(rcorr-1)) )<normalization>
model = Sequential()
model.add(Bidirectional(CuDNNLSTM(64, return_sequences=True),
input_shape=(max_text_len, 300)))
 model.add(Bidirectional(CuDNNLSTM(64)))
 model.add(Dense(1, activation="sigmoid"))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )
confusion_matrix = pd.crosstab(train["Survived"], train["Sex"] ).as_matrix()
cramers_v(confusion_matrix )<compute_test_metric>
batch_size = 128
mg = batch_gen(train_df, batch_size, emb, max_text_len)
 model.fit_generator(mg, epochs=20,
steps_per_epoch=1000,
validation_data=(val_X, val_y),
verbose=True )
scipy.stats.spearmanr(train["Survived"], train["Sex"] )<normalization>
predicted_val_prob = model.predict(val_X ).flatten()
best_thr, best_f1 = 0., 0.
 for clf_thr in np.arange(0.1, 0.501, 0.01):
clf_thr = np.round(clf_thr, 2)
 predicted_val_bin =(np.array(predicted_val_prob)> clf_thr ).astype(np.int)
 f1_val = metrics.f1_score(val_y, predicted_val_bin)
 if best_f1 <= f1_val:
best_f1, best_thr = f1_val, clf_thr
print('F1 score at threshold {} is {:1.3f}'.format(clf_thr, f1_val))
print('Best classification threshold on validation set is {}, F1 is {:1.3f}'.format(best_thr, best_f1))
confusion_matrix = pd.crosstab(train["Survived"], train["Pclass"] ).as_matrix()
cramers_v(confusion_matrix )<compute_test_metric>
all_preds_prob = []
batch_size_test = 256
for x in tqdm(batch_gen_test(dl.test_df, emb, batch_size_test, max_text_len)) :
all_preds_prob.extend(model.predict(x ).flatten())
 all_preds_prob_bin =(np.array(all_preds_prob)> best_thr ).astype(np.int )
<normalization><EOS> 
submit_df = pd.DataFrame({"qid": dl.test_df["qid"], "prediction": all_preds_prob_bin})
 submit_df.to_csv("submission.csv", index=False )