Kaggle competition: Quora Insincere Questions Classification
learner.freeze_to(-3 )<train_model>
from tensorflow.keras.layers import *
from tensorflow.keras.models import *
from tensorflow.keras.optimizers import *
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from tensorflow.keras import backend as K
from tensorflow.keras.layers import Layer
from tensorflow.keras import initializers, regularizers, constraints, optimizers, layers
import gc, re
from sklearn import metrics
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from sklearn.metrics import f1_score, roc_auc_score
learner.fit_one_cycle(1, max_lr=slice(lr*0.95**num_groups, lr), moms=(0.8, 0.9))<save_model>
def clean_text(x):
x = str(x)
 for punct in "/-'":
x = x.replace(punct, ' ')
 for punct in '&':
x = x.replace(punct, f' {punct} ')
 for punct in '?!.,"
x = x.replace(punct, '')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispell_dict = {'colour':'color',
'centre':'center',
'didnt':'did not',
'doesnt':'does not',
'isnt':'is not',
'shouldnt':'should not',
'favourite':'favorite',
'travelling':'traveling',
'counselling':'counseling',
'theatre':'theater',
'cancelled':'canceled',
'labour':'labor',
'organisation':'organization',
'wwii':'world war 2',
'citicise':'criticize',
'instagram': 'social medium',
'whatsapp': 'social medium',
'snapchat': 'social medium'
}
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text )
learner.save('third_cycle' )<load_pretrained>
def load_and_preprocess_data(max_features=50000, maxlen=70):
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: clean_text(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_numbers(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: clean_numbers(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
np.random.seed(2018)
 trn_idx = np.random.permutation(len(train_X))
train_X = train_X[trn_idx]
train_y = train_y[trn_idx]
return train_X, test_X, train_y, tokenizer.word_index
seed_all(seed)
 learner.load('third_cycle');<load_pretrained>
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
learner.unfreeze()<train_model>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight('{}_W'.format(self.name), shape=(input_shape[-1].value,),
initializer=self.init,
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight('{}_b'.format(self.name), shape=(input_shape[1].value,),
initializer='zero',
regularizer=self.b_regularizer,
constraint=self.b_constraint 
)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
learner.fit_one_cycle(2, max_lr=slice(lr*0.95**num_groups, lr), moms=(0.8, 0.9))<predict_on_test>
def model_lstm_attention(embedding_matrix, maxlen=70, max_features=50000, units=64):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(units*2, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(units, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(units, activation='relu' )(x)
 outp= Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer=Adam() , metrics=['accuracy'])
 return model
def model_gru_attention(embedding_matrix, maxlen=70, max_features=50000, units=64):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNGRU(units*2, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(units, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(units, activation='relu' )(x)
 outp= Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer=Adam() , metrics=['accuracy'])
 return model
def model_gru_attention_3(embedding_matrix, maxlen=70, max_features=50000, units=64):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNGRU(units*2, kernel_initializer=initializers.glorot_uniform(seed = SEED), return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(units, kernel_initializer=initializers.glorot_uniform(seed = SEED), return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(units//2, kernel_initializer=initializers.glorot_uniform(seed = SEED), return_sequences=True))(x)
 x = Attention(maxlen )(x)
 outp= Dense(1, kernel_initializer=initializers.he_uniform(seed=SEED), activation='sigmoid' )(x)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer=Adam() , metrics=['accuracy'])
 return model
learner.predict('This is the best movie of 2020' )<predict_on_test>
def train_pred(model, train_X, train_y, val_X, val_y, epochs=2):
for e in range(epochs):
model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y))
pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)
 best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
if score > best_score:
best_thresh = thresh
best_score = score
print("Val F1 Score: {:.4f}".format(best_score))
pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)
 print('='*100)
 return pred_val_y, pred_test_y, best_score
learner.predict('This is the worst movie of 2020' )<load_pretrained>
embed_size = 300
max_features = 100000
maxlen = 40
path = '/kaggle/working'
export_learner = load_learner(path, file = 'transformer.pkl' )<predict_on_test>
start_time = time.time()
train_X, test_X, train_y, word_index = load_and_preprocess_data(max_features=max_features, maxlen=maxlen)
 total_time =(time.time() - start_time)/60.0
print("Took {0} minutes".format(total_time))
export_learner.predict('This is the worst movie of 2020' )<predict_on_test>
start_time = time.time()
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_3 = load_para(word_index)
 total_time =(time.time() - start_time)/60.0
print("Took {0} minutes".format(total_time))
def get_preds_as_nparray(ds_type)-> np.ndarray:

 preds = learner.get_preds(ds_type)[0].detach().cpu().numpy()
sampler = [i for i in databunch.dl(ds_type ).sampler]
reverse_sampler = np.argsort(sampler)
 return preds[reverse_sampler, :]
test_preds = get_preds_as_nparray(DatasetType.Test )<save_to_csv>
embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_3], axis = 0)
 print(np.shape(embedding_matrix))
sample_submission = pd.read_csv(DATA_ROOT / 'sampleSubmission.csv')
 sample_submission['Sentiment'] = np.argmax(test_preds,axis=1)
 sample_submission.to_csv("predictions.csv", index=False )<save_to_csv>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
def create_download_link(title = "Download CSV file", filename = "data.csv"):
html = '<a href={filename}>{title}</a>'
html = html.format(title=title,filename=filename)
 return HTML(html)
 create_download_link(filename='predictions.csv' )<string_transform>
train_meta = np.zeros(train_y.shape)
 test_meta = np.zeros(test_X.shape[0])
 splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=SEED ).split(train_X, train_y))
for idx,(train_idx, valid_idx)in enumerate(splits):
X_train = train_X[train_idx]
y_train = train_y[train_idx]
X_val = train_X[valid_idx]
y_val = train_y[valid_idx]
tmp_result = []
model = model_gru_attention_3(embedding_matrix, maxlen=maxlen, max_features=max_features, units=64)
 pred_val_y, pred_test_y, best_score = train_pred(model, X_train, y_train, X_val, y_val, epochs = 2)
 tmp_result.append([pred_val_y, pred_test_y, best_score])
 model = model_gru_attention(embedding_matrix, maxlen=maxlen, max_features=max_features, units=64)
 pred_val_y, pred_test_y, best_score = train_pred(model, X_train, y_train, X_val, y_val, epochs = 2)
 tmp_result.append([pred_val_y, pred_test_y, best_score])
 model = model_lstm_attention(embedding_matrix_1, maxlen=maxlen, max_features=max_features, units=64)
 pred_val_y, pred_test_y, best_score = train_pred(model, X_train, y_train, X_val, y_val, epochs = 2)
 tmp_result.append([pred_val_y, pred_test_y, best_score])
 pred_val_y, pred_test_y, best_score = np.mean(tmp_result, axis=0)
 print("Mean Result: ", best_score)
 train_meta[valid_idx] = pred_val_y.reshape(-1)
 test_meta += pred_test_y.reshape(-1)/ len(splits )
<create_dataframe><EOS> 
search_result = threshold_search(train_y, train_meta)
 print(search_result)
 sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.prediction = test_meta > search_result['threshold']
sub.to_csv("submission.csv", index=False )