Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<feature_engineering>
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
import os
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
n=21
commits_df.loc[n,'commit_num'] = 31
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'GaussianNoise_stddev'] = 0.21
commits_df.loc[n,'LB_score'] = -6.8093<feature_engineering>
train_df = pd.read_csv(".. /input/train.csv")
 train_df, val_df = train_test_split(train_df, test_size=0.1 )
n=22
commits_df.loc[n,'commit_num'] = 32
commits_df.loc[n,'Dropout_model'] = 0.24
commits_df.loc[n,'FVC_weight'] = 0.14
commits_df.loc[n,'GaussianNoise_stddev'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8106<feature_engineering>
embeddings_index = {}
f = open('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt')
 for line in tqdm(f):
values = line.split(" ")
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
f.close()
print('Found %s word vectors.' % len(embeddings_index))
commits_df['LB_score'] = pd.to_numeric(commits_df['LB_score'])
 commits_df['max'] = 0
commits_df.loc[commits_df['LB_score'].idxmax() , 'max'] = 1<feature_engineering>
def text_to_array(text):
empyt_emb = np.zeros(300)
 text = text[:-1].split() [:100]
embeds = [embeddings_index.get(x, empyt_emb)for x in text]
embeds+= [empyt_emb] *(100 - len(embeds))
return np.array(embeds)
 val_vects = np.array([text_to_array(X_text)for X_text in tqdm(val_df["question_text"][:5000])])
 val_y = np.array(val_df["target"][:5000])
commits_df['seed'] = 42
commits_df.loc[commits_df['commit_num'] == 23, 'seed'] = 0<feature_engineering>
batch_size = 128
def batch_gen(train_df):
n_batches = math.ceil(len(train_df)/ batch_size)
 while True:
train_df = train_df.sample(frac=1.)
for i in range(n_batches):
texts = train_df.iloc[i*batch_size:(i+1)*batch_size, 1]
text_arr = np.array([text_to_array(text)for text in texts])
 yield text_arr, np.array(train_df["target"][i*batch_size:(i+1)*batch_size])
commits_df['GaussianNoise_stddev'] = 0.2
commits_df.loc[commits_df['commit_num'] == 28, 'GaussianNoise_stddev'] = 0.25<sort_values>
from keras.models import Sequential, Input
from keras.models import Model
from keras.layers import concatenate, GlobalAveragePooling1D, GlobalMaxPooling1D, CuDNNLSTM, CuDNNGRU, Dense, Bidirectional, SpatialDropout1D, Conv1D
commits_df.sort_values(by=['LB_score'], ascending = False )<load_from_csv>
dr = 0.25
units = 96
inp = Input(shape =(( 100, 300)))
 x1 = SpatialDropout1D(dr )(inp)
 x = Bidirectional(CuDNNGRU(units, return_sequences = True))(x1)
 x = Conv1D(64, kernel_size = 2, padding = "valid", kernel_initializer = "he_uniform" )(x)
 y = Bidirectional(CuDNNLSTM(units, return_sequences = True))(x1)
 y = Conv1D(64, kernel_size = 2, padding = "valid", kernel_initializer = "he_uniform" )(y)
 avg_pool1 = GlobalAveragePooling1D()(x)
 max_pool1 = GlobalMaxPooling1D()(x)
 avg_pool2 = GlobalAveragePooling1D()(y)
 max_pool2 = GlobalMaxPooling1D()(y)
 x = concatenate([avg_pool1, max_pool1, avg_pool2, max_pool2])
 x = Dense(1, activation = "sigmoid" )(x)
 model = Model(inputs = inp, outputs = x)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'] )
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv' )<concatenate>
mg = batch_gen(train_df)
 model.fit_generator(mg, epochs=30,
steps_per_epoch=1000,
validation_data=(val_vects, val_y),
verbose=True )
def get_tab(df):
vector = [(df.Age.values[0] - 30)/ 30]
if df.Sex.values[0] == 'male':
vector.append(0)
 else:
vector.append(1)
 if df.SmokingStatus.values[0] == 'Never smoked':
vector.extend([0,0])
 elif df.SmokingStatus.values[0] == 'Ex-smoker':
vector.extend([1,1])
 elif df.SmokingStatus.values[0] == 'Currently smokes':
vector.extend([0,1])
 else:
vector.extend([1,0])
 return np.array(vector )<feature_engineering>
batch_size = 256
def batch_gen(test_df):
n_batches = math.ceil(len(test_df)/ batch_size)
 for i in range(n_batches):
texts = test_df.iloc[i*batch_size:(i+1)*batch_size, 1]
text_arr = np.array([text_to_array(text)for text in texts])
 yield text_arr
test_df = pd.read_csv(".. /input/test.csv")
 all_preds = []
for x in tqdm(batch_gen(test_df)) :
all_preds.extend(model.predict(x ).flatten() )
A = {}
TAB = {}
P = []
for i, p in tqdm(enumerate(train.Patient.unique())) :
sub = train.loc[train.Patient == p, :]
fvc = sub.FVC.values
weeks = sub.Weeks.values
c = np.vstack([weeks, np.ones(len(weeks)) ] ).T
a, b = np.linalg.lstsq(c, fvc)[0]
A[p] = a
TAB[p] = get_tab(sub)
 P.append(p )<feature_engineering>
all_preds_val = []
for x in tqdm(batch_gen(val_df)) :
all_preds_val.extend(model.predict(x ).flatten() )
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a<choose_model_class>
val_y = val_df["target"].values
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(512, 512, 1), model_class=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(4,))
x2 = tf.keras.layers.GaussianNoise(GaussianNoise_stddev )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(Dropout_model )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 weights = [w for w in os.listdir('.. /input/osic-model-weights')if model_class in w][0]
model.load_weights('.. /input/osic-model-weights/' + weights)
 return model
model_classes = ['b5']
models = [build_model(shape=(512, 512, 1), model_class=m)for m in model_classes]
print('Number of models: ' + str(len(models)) )<split>
from sklearn.metrics import f1_score
tr_p, vl_p = train_test_split(P, shuffle=True, train_size = 0.8 )<compute_test_metric>
y_val =(np.array(all_preds_val)> 0.1 ).astype(np.int)
 f1_score(val_y, y_val )
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric )<define_variables>
f1_score(val_y, y_val )
subs = []
for model in models:
metric = []
for q in tqdm(range(1, 10)) :
m = []
for p in vl_p:
x = []
tab = []
if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:
continue
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))
tab.append(get_tab(train.loc[train.Patient == p, :]))
if len(x)< 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q / 10)
 percent_true = train.Percent.values[train.Patient == p]
fvc_true = train.FVC.values[train.Patient == p]
weeks_true = train.Weeks.values[train.Patient == p]
fvc = a *(weeks_true - weeks_true[0])+ fvc_true[0]
percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])
 m.append(score(fvc_true, fvc, percent))
print(np.mean(m))
metric.append(np.mean(m))
q =(np.argmin(metric)+ 1)/ 10
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub )<prepare_output>
score = 0
thresh =.5
for i in np.arange(0.1, 0.991, 0.01):
y_val =(np.array(all_preds_val)> i ).astype(np.int)
 temp_score = f1_score(val_y, y_val)
 if(temp_score > score):
score = temp_score
thresh = i
print("CV: {}, Threshold: {}".format(score, thresh))
<save_to_csv><EOS> 
y_te =(np.array(all_preds)> thresh ).astype(np.int)
 submit_df = pd.DataFrame({"qid": test_df["qid"], "prediction": y_te})
 submit_df.to_csv("submission.csv", index=False )