Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<categorify>
tqdm.pandas()
X_train_keras = train_imagesRaw.reshape(-1,28,28,1)
 X_test_keras = test_imagesRaw.reshape(-1,28,28,1)
 print("X_train_keras",X_train_keras.shape)
 print("X_test_keras",X_test_keras.shape)
 train_labels_keras = tf.keras.utils.to_categorical(train_labelsRaw)
 test_labels_keras = tf.keras.utils.to_categorical(test_labelsRaw)
 print("train_labels_keras ",train_labels_keras.shape)
 print("test_labels_keras ", test_labels_keras.shape )<concatenate>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print("Train shape : ", train.shape)
 print("Test shape : ", test.shape )
train_images = np.concatenate(( X_train_keras,X_train,X_test_keras), axis=0)
 print("new Concatenated train_images ", train_images.shape)
 print("_"*50)
 train_labels = np.concatenate(( train_labels_keras,y_train,test_labels_keras), axis=0)
 print("new Concatenated train_labels ", train_labels.shape )<normalization>
train["question_text"] = train["question_text"].str.lower()
test["question_text"] = test["question_text"].str.lower()
scale = np.max(train_images)
 train_images /= scale
X_test /= scale
print("Max: {}".format(scale))<split>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
X_train, X_val, y_train, y_val = train_test_split(train_images, train_labels, test_size=0.10 )<train_model>
train["question_text"] = train["question_text"].apply(lambda x: clean_text(x))
test["question_text"] = test["question_text"].apply(lambda x: clean_text(x))
input_size = X_train.shape
n_logits = y_train.shape[1]
print("Input: {}".format(input_size))
print("Output: {}".format(n_logits))<choose_model_class>
train_df, val_df = train_test_split(train, test_size=0.1, random_state=2018)
 embed_size = 300
max_features = None
maxlen = 72
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=None, filters='')
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
sub = test[['qid']]
def build_model(hp):

 num_layers = hp.Int('num_layers', min_value=2, max_value=16, step=2)
 lr = hp.Choice('learning_rate', [1e-3, 5e-4])
 inputs = layers.Input(shape=(28, 28, 1))
x = inputs
for idx in range(num_layers):
idx = str(idx)
 filters = hp.Int('filters_' + idx, 32, 256, step=32, default=64)
 x = layers.Conv2D(filters=filters, kernel_size=3, padding='same',
activation='relu' )(x)
 if x.shape[1] >= 8:
pool_type = hp.Choice('pool_' + idx, values=['max', 'avg'])
 if pool_type == 'max':
x = layers.MaxPooling2D(2 )(x)
 elif pool_type == 'avg':
x = layers.AveragePooling2D(2 )(x)
 x = layers.Flatten()(x)
 x = layers.Dense(256, activation='relu' )(x)
 x = layers.Dense(256, activation='relu' )(x)
 x = layers.Dense(256, activation='relu' )(x)
 x = layers.Dropout(0.5 )(x)
 outputs = layers.Dense(n_logits, activation='softmax' )(x)
 model = keras.Model(inputs, outputs)
 model.compile(optimizer=Adam(lr),
loss='categorical_crossentropy',
metrics=['accuracy'])
 return model<find_best_params>
del train, test
gc.collect()
tuner = RandomSearch(
 build_model,
objective='val_accuracy',
max_trials=8,
executions_per_trial=3,
directory='my_dir',
project_name='mnist')
 tuner.search_space_summary()<compute_test_metric>
EMBEDDING_FILE_GLOVE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE_GLOVE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
np.random.seed(10)
 word_index = tokenizer.word_index
max_features = len(word_index)+1
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
tuner.search(X_train, y_train,
epochs=30,
validation_data=(X_val, y_val))<save_to_csv>
class AdamW(Optimizer):
def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999, weight_decay=1e-4,
epsilon=1e-8, decay=0., **kwargs):
super(AdamW, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.lr = K.variable(lr, name='lr')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.wd = K.variable(weight_decay, name='weight_decay')
 self.epsilon = epsilon
self.initial_decay = decay
@interfaces.legacy_get_updates_support
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
wd = self.wd
lr = self.lr
if self.initial_decay > 0:
lr *=(1./(1.+ self.decay * K.cast(self.iterations,
K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx())+ 1
lr_t = lr *(K.sqrt(1.- K.pow(self.beta_2, t)) / 
(1.- K.pow(self.beta_1, t)))
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
self.weights = [self.iterations] + ms + vs
for p, g, m, v in zip(params, grads, ms, vs):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 p_t = p - lr_t * m_t /(K.sqrt(v_t)+ self.epsilon)- lr * wd * p
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
def get_config(self):
config = {'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.wd)) ,
'epsilon': self.epsilon}
base_config = super(AdamW, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
predictions_vector = model.predict(X_test, verbose=0)
 predictions = np.argmax(predictions_vector,axis=1)
 pd.DataFrame({"ImageId": list(range(1,len(predictions)+1)) , "Label": predictions} ).to_csv("preds.csv", index=False, header=True )<import_modules>
%%time
K.clear_session()
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(rate=0.2 )(x)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True,
kernel_initializer=glorot_normal(seed=12300), recurrent_initializer=orthogonal(gain=1.0, seed=10000)) )(x)
 x1 = Conv1D(filters=96, kernel_size=1, padding='same', activation='relu', kernel_initializer=glorot_normal(1000))(x)
 x2 = Conv1D(filters=72, kernel_size=2, padding='same', activation='relu', kernel_initializer=glorot_normal(2000))(x)
 x3 = Conv1D(filters=48, kernel_size=3, padding='same', activation='relu', kernel_initializer=glorot_normal(3000))(x)
 x4 = Conv1D(filters=24, kernel_size=5, padding='same', activation='relu', kernel_initializer=glorot_normal(4000))(x)
 x1 = GlobalMaxPool1D()(x1)
 x2 = GlobalMaxPool1D()(x2)
 x3 = GlobalMaxPool1D()(x3)
 x4 = GlobalMaxPool1D()(x4)
 merge1 = concatenate([x1, x2, x3, x4])
 x = Dense(200, activation="relu", kernel_initializer=he_uniform(seed=12300))(merge1)
 x = Dropout(0.22 )(x)
 x = BatchNormalization()(x)
 x = Dense(200, activation="relu", kernel_initializer=he_uniform(seed=12300))(x)
 x = Dropout(0.22 )(x)
 x = BatchNormalization()(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=AdamW(weight_decay=0.08),)
 print(model.summary() )
import pandas as pd<import_modules>
import pandas as pd<load_from_csv>
%%time
filepath="weights_best.h5"
checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=2, save_best_only=True, mode='min')
 callbacks = [checkpoint]
model.fit(train_X, train_y, batch_size=512, epochs=5,
validation_data=(val_X, val_y), verbose=2, callbacks=callbacks,
)
<feature_engineering><EOS> 
model.load_weights(filepath)
 pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=2)
 pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=2)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(val_y,(pred_glove_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh)
 pred_test_y =(pred_glove_test_y>best_thresh ).astype(int)
 sub['prediction'] = pred_test_y
sub.to_csv("submission.csv", index=False )