Kaggle competition: Quora Insincere Questions Classification
train_text = train[['premise', 'hypothesis']].values.tolist()
test_text = test[['premise', 'hypothesis']].values.tolist()<categorify>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train.shape)
 print("Test shape : ",test.shape )
train_encoded = tokenizer.batch_encode_plus(
 train_text,
pad_to_max_length=True,
max_length=MAX_LEN
)<categorify>
EMBED_SIZE = 300
MAX_FEATURES = 100000
MAXLEN = 40
test_encoded = tokenizer.batch_encode_plus(
 test_text,
pad_to_max_length=True,
max_length=MAX_LEN
)<string_transform>
def build_vocab(texts):
sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
vocab = tokenizer.get_vocab()
print(vocab['
'])
 print(vocab['▁and'])
 print(vocab['▁these'])
 print(vocab['▁comments'])
 print(vocab['▁were'])
 print(vocab['▁considered'])
 print(vocab['▁in'])
 print(vocab['▁formula'])
 print(vocab['ting'])
 print(vocab['▁the'])
 print(vocab['▁inter'])
 print(vocab['im'])
 print(vocab['▁rules'])
 print(vocab['.'] )<split>
df = pd.concat([train ,test], sort=False)
 vocab = build_vocab(df['question_text'] )
x_train, x_valid, y_train, y_valid = train_test_split(
 train_encoded['input_ids'],
train.label.values,
test_size=0.2,
random_state=2020
)<prepare_x_and_y>
def load_embed(file):
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding='latin'))
return embeddings_index
x_test = test_encoded['input_ids']<train_model>
glove = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
embed_glove = load_embed(glove)
 len(embed_glove )
auto = tf.data.experimental.AUTOTUNE
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(auto)
 )<train_model>
def load_glove_matrix(word_index, embeddings_index):
all_embs = np.stack(embeddings_index.values())
 emb_mean, emb_std = all_embs.mean() , all_embs.std()
EMBED_SIZE = all_embs.shape[1]
nb_words = min(MAX_FEATURES, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, EMBED_SIZE))
for word, i in word_index.items() :
if i >= MAX_FEATURES:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix
valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(auto)
 )<create_dataframe>
def check_coverage(vocab, embeddings_index):
known_words = {}
unknown_words = {}
nb_known_words = 0
nb_unknown_words = 0
for word in vocab.keys() :
try:
known_words[word] = embeddings_index[word]
nb_known_words += vocab[word]
except:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
pass
print('Found embeddings for {:.3%} of vocabulary'.format(len(known_words)/len(vocab)))
 print('Found embeddings for {:.3%} of all text'.format(nb_known_words/(nb_known_words + nb_unknown_words)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words
test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<load_pretrained>
print("Glove: ")
 oov_glove = check_coverage(vocab, embed_glove )
with strategy.scope() :
backbone = TFAutoModel.from_pretrained(MODEL_NAME )<choose_model_class>
def add_lower(embedding, vocab):
count = 0
for word in vocab:
if word in embedding and word.lower() not in embedding:
embedding[word.lower() ] = embedding[word]
count += 1
print(f"Added {count} words to embedding" )
with strategy.scope() :
x_input = tf.keras.Input(shape=(MAX_LEN,), dtype=tf.int32, name="input_ids")
 x = backbone(x_input)[0]
x = x[:, 0, :]
x = tf.keras.layers.Dense(3, activation='softmax' )(x)
 model = tf.keras.models.Model(inputs=x_input, outputs=x )<choose_model_class>
train['question_text'] = train['question_text'].apply(lambda x: x.lower())
 test['question_text'] = test['question_text'].apply(lambda x: x.lower() )
model.compile(
 tf.keras.optimizers.Adam(lr=RATE),
loss='sparse_categorical_crossentropy',
metrics=['accuracy']
)<train_model>
print("Glove: ")
 oov_glove = check_coverage(vocab, embed_glove)
 add_lower(embed_glove, vocab)
 oov_glove = check_coverage(vocab, embed_glove )
steps = len(x_train)// BATCH_SIZE
history = model.fit(
 train_dataset,
validation_data=valid_dataset,
epochs=EPOCHS,
steps_per_epoch=steps,
)<predict_on_test>
contraction_mapping = {
"ain't": "is not",
"aren't": "are not",
"can't": "cannot",
"'cause": "because",
"could've": "could have",
"couldn't": "could not",
"didn't": "did not",
"doesn't": "does not",
"don't": "do not",
"hadn't": "had not",
"hasn't": "has not",
"haven't": "have not",
"he'd": "he would",
"he'll": "he will",
"he's": "he is",
"how'd": "how did",
"how'd'y": "how do you",
"how'll": "how will",
"how's": "how is",
"I'd": "I would",
"I'd've": "I would have",
"I'll": "I will",
"I'll've": "I will have",
"I'm": "I am",
"I've": "I have",
"i'd": "i would",
"i'd've": "i would have",
"i'll": "i will",
"i'll've": "i will have",
"i'm": "i am",
"i've": "i have",
"isn't": "is not",
"it'd": "it would",
"it'd've": "it would have",
"it'll": "it will",
"it'll've": "it will have",
"it's": "it is",
"let's": "let us",
"ma'am": "madam",
"mayn't": "may not",
"might've": "might have",
"mightn't": "might not",
"mightn't've": "might not have",
"must've": "must have",
"mustn't": "must not",
"mustn't've": "must not have",
"needn't": "need not",
"needn't've": "need not have",
"o'clock": "of the clock",
"oughtn't": "ought not",
"oughtn't've": "ought not have",
"shan't": "shall not",
"sha'n't": "shall not",
"shan't've": "shall not have",
"she'd": "she would",
"she'd've": "she would have",
"she'll": "she will",
"she'll've": "she will have",
"she's": "she is",
"should've": "should have",
"shouldn't": "should not",
"shouldn't've": "should not have",
"so've": "so have",
"so's": "so as",
"this's": "this is",
"that'd": "that would",
"that'd've": "that would have",
"that's": "that is",
"there'd": "there would",
"there'd've": "there would have",
"there's": "there is",
"here's": "here is",
"they'd": "they would",
"they'd've": "they would have",
"they'll": "they will",
"they'll've": "they will have",
"they're": "they are",
"they've": "they have",
"to've": "to have",
"wasn't": "was not",
"we'd": "we would",
"we'd've": "we would have",
"we'll": "we will",
"we'll've": "we will have",
"we're": "we are",
"we've": "we have",
"weren't": "were not",
"what'll": "what will",
"what'll've": "what will have",
"what're": "what are",
"what's": "what is",
"what've": "what have",
"when's": "when is",
"when've": "when have",
"where'd": "where did",
"where's": "where is",
"where've": "where have",
"who'll": "who will",
"who'll've": "who will have",
"who's": "who is",
"who've": "who have",
"why's": "why is",
"why've": "why have",
"will've": "will have",
"won't": "will not",
"won't've": "will not have",
"would've": "would have",
"wouldn't": "would not",
"wouldn't've": "would not have",
"y'all": "you all",
"y'all'd": "you all would",
"y'all'd've": "you all would have",
"y'all're": "you all are",
"y'all've": "you all have",
"you'd": "you would",
"you'd've": "you would have",
"you'll": "you will",
"you'll've": "you will have",
"you're": "you are",
"you've": "you have",
"who'd": "who would",
"who're": "who are",
"'re": " are",
"tryin'": "trying",
"doesn'": "does not",
'howdo': 'how do',
'whatare': 'what are',
'howcan': 'how can',
'howmuch': 'how much',
'howmany': 'how many',
'whydo': 'why do',
'doI': 'do I',
'theBest': 'the best',
'howdoes': 'how does',
}
len(contraction_mapping )
test_preds = model.predict(test_dataset, verbose=1)
 submission['prediction'] = test_preds.argmax(axis=1 )<save_to_csv>
def known_contractions(embed):
known = []
for contract in contraction_mapping:
if contract in embed:
known.append(contract)
 return known
print(known_contractions(embed_glove))
submission.to_csv('submission.csv', index=False )<save_to_csv>
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
submission.to_csv('submission.csv', index=False )<feature_engineering>
train['question_text'] = train['question_text'].apply(lambda x: clean_contractions(x, contraction_mapping))
test['question_text'] = test['question_text'].apply(lambda x: clean_contractions(x, contraction_mapping))
os.environ["WANDB_API_KEY"] = "0"<define_variables>
df = pd.concat([train ,test], sort=False)
 vocab = build_vocab(df['question_text'])
 print("Glove: ")
 oov_glove = check_coverage(vocab, embed_glove )
SEED=42
max_len = 48
EPOCHS = 3
BATCH_SIZE = 64
LR = 1e-5<import_modules>
punct_mapping = "/-'?!.,
punct_mapping += '©^®` <→°€™› ♥←×§″′Â█½à…“★”–●â►−¢²¬░¶↑±¿▾═¦║―¥▓—‹─▒：¼⊕▼▪†■’▀¨▄♫☆é¯♦¤▲è¸¾Ã⋅‘∞∙）↓、│（»，♪╩╚³・╦╣╔╗▬❤ïØ¹≤‡√'
def unknown_punct(embed, punct):
unknown = ''
for p in punct:
if p not in embed:
unknown += p
unknown += ' '
return unknown
print(unknown_punct(embed_glove, punct_mapping))
from transformers import BertTokenizer, TFBertModel,TFAutoModel,TFXLMRobertaModel, AutoTokenizer
import matplotlib.pyplot as plt
import tensorflow as tf<load_from_csv>
puncts = {"‘": "'", "´": "'", "°": "", "€": "e", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', '…': ' '}
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 return text
train = pd.read_csv(".. /input/contradictory-my-dear-watson/train.csv" )<load_pretrained>
train['question_text'] = train['question_text'].apply(lambda x: clean_special_chars(x, punct_mapping, puncts))
test['question_text'] = test['question_text'].apply(lambda x: clean_special_chars(x, punct_mapping, puncts))
model_name = 'jplu/tf-xlm-roberta-large'
tokenizer = AutoTokenizer.from_pretrained(model_name)
<categorify>
df = pd.concat([train ,test], sort=False)
 vocab = build_vocab(df['question_text'])
 print("Glove: ")
 oov_glove = check_coverage(vocab, embed_glove )
def encode_sentence(s):
tokens = list(tokenizer.tokenize(s))
tokens.append('[SEP]')
 return tokenizer.convert_tokens_to_ids(tokens )<categorify>
train, val = train_test_split(train, test_size=0.2, random_state=42 )
encode_sentence("I love machine learning" )<categorify>
xtrain = train['question_text'].fillna('_na_' ).values
xval = val['question_text'].fillna('_na_' ).values
xtest = test['question_text'].fillna('_na_' ).values
def bert_encode(hypotheses, premises, tokenizer):
num_examples = len(hypotheses)
 sentence1 = tf.ragged.constant([
encode_sentence(s)
 for s in np.array(hypotheses)])
 sentence2 = tf.ragged.constant([
encode_sentence(s)
 for s in np.array(premises)])
 cls = [tokenizer.convert_tokens_to_ids(['[CLS]'])]*sentence1.shape[0]
input_word_ids = tf.concat([cls, sentence1, sentence2], axis=-1)
 input_mask = tf.ones_like(input_word_ids ).to_tensor()
type_cls = tf.zeros_like(cls)
 type_s1 = tf.zeros_like(sentence1)
 type_s2 = tf.ones_like(sentence2)
 input_type_ids = tf.concat(
 [type_cls, type_s1, type_s2], axis=-1 ).to_tensor()
inputs = {
'input_word_ids': input_word_ids.to_tensor() ,
'input_mask': input_mask,
'input_type_ids': input_type_ids}
return inputs<categorify>
tokenizer = Tokenizer(num_words=MAX_FEATURES)
 tokenizer.fit_on_texts(list(xtrain))
xtrain = tokenizer.texts_to_sequences(xtrain)
 xval = tokenizer.texts_to_sequences(xval)
 xtest = tokenizer.texts_to_sequences(xtest )
train_input = bert_encode(train.premise.values, train.hypothesis.values, tokenizer )<categorify>
xtrain = pad_sequences(xtrain, maxlen=MAXLEN)
 xval = pad_sequences(xval, maxlen=MAXLEN)
 xtest = pad_sequences(xtest, maxlen=MAXLEN )
def build_model() :
bert_encoder = TFXLMRobertaModel.from_pretrained(model_name)
 input_word_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 input_type_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_type_ids")
 embedding = bert_encoder([input_word_ids, input_mask, input_type_ids])[0]
output = tf.keras.layers.Dense(3, activation='softmax' )(embedding[:,0,:])
 model = tf.keras.Model(inputs=[input_word_ids, input_mask, input_type_ids], outputs=output)
 model.compile(tf.keras.optimizers.Adam(lr=LR), loss='sparse_categorical_crossentropy', metrics=['accuracy'])
 return model<choose_model_class>
ytrain = train['target'].values
yval = val['target'].values
eas = EarlyStopping(monitor='val_loss', min_delta=0.001, patience=3,
verbose=1, mode='min', baseline=None, restore_best_weights=True )<train_model>
np.random.seed(42)
 trn_idx = np.random.permutation(len(xtrain))
val_idx = np.random.permutation(len(xval))
xtrain = xtrain[trn_idx]
ytrain = ytrain[trn_idx]
xval = xval[val_idx]
yval = yval[val_idx]
embedding_matrix_glove = load_glove_matrix(tokenizer.word_index, embed_glove )
model.fit(train_input, train.label.values, epochs = EPOCHS, verbose = 1, batch_size = BATCH_SIZE, validation_split = 0.2, callbacks = [eas])
<load_from_csv>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
test = pd.read_csv(".. /input/contradictory-my-dear-watson/test.csv")
 test_input = bert_encode(test.premise.values, test.hypothesis.values, tokenizer )<predict_on_test>
def f1(y_true, y_pred):
def recall(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true*y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives/(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true*y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives/(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
predictions = [np.argmax(i)for i in model.predict(test_input)] 
<create_dataframe>
def model_lstm_att(embedding_matrix):
inp = Input(shape=(MAXLEN,))
x = Embedding(MAX_FEATURES, EMBED_SIZE, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(32, return_sequences=True))(x)
 att = Attention(MAXLEN )(x)
 y = Dense(32, activation='relu' )(att)
 y = Dropout(0.1 )(y)
 outp = Dense(1, activation='sigmoid' )(y)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['binary_accuracy'])
 return model
submission = test.id.copy().to_frame()
submission['prediction'] = predictions<save_to_csv>
def train_pred(model, epochs=2):
for e in range(epochs):
model.fit(xtrain, ytrain, batch_size=512, epochs=3, validation_data=(xval, yval))
pred_val_y = model.predict([xval], batch_size=1024, verbose=1)
 best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(yval,(pred_val_y > thresh ).astype(int))
if score > best_score:
best_thresh = thresh
best_score = score
print("Val puntuacion F1: {:.4f}".format(best_score))
pred_test_y = model.predict([xtest], batch_size=1024, verbose=1)
 return pred_val_y, pred_test_y, best_score
submission.to_csv("submission.csv", index = False )<feature_engineering>
paragram = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
embedding_matrix_para = load_glove_matrix(tokenizer.word_index, load_embed(paragram))
os.environ["WANDB_API_KEY"] = "0"
!pip uninstall -y transformers
!pip install transformers
!pip install nlp
strategy = None<load_from_csv>
embedding_matrix = np.mean([embedding_matrix_glove, embedding_matrix_para], axis=0 )
original_train = pd.read_csv(".. /input/contradictory-my-dear-watson/train.csv")
 original_train = sklearn.utils.shuffle(original_train)
 original_train = sklearn.utils.shuffle(original_train)
 validation_ratio = 0.2
nb_valid_examples = max(1, int(len(original_train)* validation_ratio))
original_valid = original_train[:nb_valid_examples]
original_train = original_train[nb_valid_examples:]<train_model>
outputs = []
pred_val_y, pred_test_y, best_score = train_pred(model_lstm, epochs=3)
 outputs.append([pred_val_y, pred_test_y, best_score, 'model_lstm_att only Glove'] )
print(f"original - training: {len(original_train)} examples")
 original_train.head(10 )<train_model>
outputs.sort(key=lambda x: x[2])
 weights = [i for i in range(1, len(outputs)+ 1)]
weights = [float(i)/ sum(weights)for i in weights]
pred_val_y = np.mean([outputs[i][0] for i in range(len(outputs)) ], axis = 0)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(yval,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("La puntuación F1 en los limites {0} y {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print(f"original - validation: {len(original_valid)} examples")
 original_valid.head(10 )<load_from_csv>
print("Mejor limite:", best_thresh, "y puntuacion F1 ", thresholds[0][1] )
original_test = pd.read_csv(".. /input/contradictory-my-dear-watson/test.csv")
 print(f"original - test: {len(original_test)} examples")
 original_test.head(10 )<create_dataframe>
pred_test_y = np.mean([outputs[i][1] for i in range(len(outputs)) ], axis = 0)
 pred_test_y =(pred_test_y > best_thresh ).astype(int )
mnli = nlp.load_dataset(path='glue', name='mnli' )<train_model>
sub = pd.read_csv('.. /input/sample_submission.csv')
 out_df = pd.DataFrame({"qid":sub["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )