Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<split>
tqdm.pandas()
Activation, Dropout, SpatialDropout1D, Embedding, Concatenate, concatenate, Reshape, Flatten,
CuDNNLSTM, CuDNNGRU, Bidirectional)
 t_start = time.time()
tokenizer = AutoTokenizer.from_pretrained(model_name)
 train_text = train[['premise', 'hypothesis']].values.tolist()
test_text = test[['premise', 'hypothesis']].values.tolist()
train_encoded = tokenizer.batch_encode_plus(
 train_text,
pad_to_max_length=True,
max_length=max_len 
)
 test_encoded = tokenizer.batch_encode_plus(
 test_text,
pad_to_max_length=True,
max_length=max_len 
)
 x_train, x_valid, y_train, y_valid = train_test_split(
 train_encoded['input_ids'], train.label.values,
test_size=0.2, random_state=2020 
)
 x_test = test_encoded['input_ids']<split>
embed_size = 300
max_features = 120000
maxlen = 70
nb_features = 6
EPOCHS = 10
auto = tf.data.experimental.AUTOTUNE
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(batch_size)
.prefetch(auto)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(batch_size)
.cache() 
.prefetch(auto)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(batch_size)
 )<categorify>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', '~', '•',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
mispell_dict = {"aren't" : "are not", "can't" : "cannot", "couldn't" : "could not", "didn't" : "did not", "doesn't" : "does not",
"don't" : "do not", "hadn't" : "had not", "hasn't" : "has not", "haven't" : "have not", "he'd" : "he would", "he'll" : "he will",
"he's" : "he is", "i'd" : "I would", "i'd" : "I had", "i'll" : "I will", "i'm" : "I am", "isn't" : "is not", "it's" : "it is",
"it'll":"it will", "i've" : "I have", "let's" : "let us", "mightn't" : "might not", "mustn't" : "must not", "shan't" : "shall not",
"she'd" : "she would", "she'll" : "she will", "she's" : "she is", "shouldn't" : "should not", "that's" : "that is", "there's" : "there is",
"they'd" : "they would", "they'll" : "they will", "they're" : "they are", "they've" : "they have", "we'd" : "we would", "we're" : "we are",
"weren't" : "were not", "we've" : "we have", "what'll" : "what will", "what're" : "what are", "what's" : "what is", "what've" : "what have",
"where's" : "where is", "who'd" : "who would", "who'll" : "who will", "who're" : "who are", "who's" : "who is", "who've" : "who have",
"won't" : "will not", "wouldn't" : "would not", "you'd" : "you would", "you'll" : "you will", "you're" : "you are", "you've" : "you have",
"'re": " are", "wasn't": "was not", "we'll":" will", "didn't": "did not", "tryin'":"trying"}
bad_words = "2 girls 1 cup, 2g1c, 4r5e, 5h1t, 5hit, a$$"
bad_words = [x.strip() for x in bad_words.split(",")]
def clean_text(x):
x = str(x)
 for word in bad_words:
x = x.replace(word, "ZZZZZZ")
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{10,}', '
x = re.sub('[0-9]{5,10}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
 def split_text(x):
x = wordninja.split(x)
 return '-'.join(x)
 def add_features(df):
df['question_text'] = df['question_text'].progress_apply(lambda x:str(x))
df['total_length'] = df['question_text'].progress_apply(len)
 df['capitals'] = df['question_text'].progress_apply(lambda comment: sum(1 for c in comment if c.isupper()))
df['caps_vs_length'] = df.progress_apply(lambda row: float(row['capitals'])/float(row['total_length']),
axis=1)
 df['num_words'] = df.question_text.str.count('\S+')
 df['num_unique_words'] = df['question_text'].progress_apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique'] = df['num_unique_words'] / df['num_words']
return df
def load_and_prec() :
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 for df in [train_df, test_df]:
df["question_text"] = df["question_text"].str.lower()
df["question_text"] = df["question_text"].progress_apply(lambda x: clean_text(x))
df["question_text"] = df["question_text"].progress_apply(lambda x: clean_numbers(x))
df["question_text"] = df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
df["question_text"] = df["question_text"].fillna("_
print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 _train = add_features(train_df)
 _test = add_features(test_df)
 ff = ['total_length', 'capitals', 'caps_vs_length', 'num_words', 'num_unique_words', 'words_vs_unique']
features = _train[ff].fillna(0)
 test_features = _test[ff].fillna(0)
 ss = StandardScaler()
ss.fit(np.vstack(( features, test_features)))
 features = ss.transform(features)
 test_features = ss.transform(test_features)
 train_X, valid_X, train_y, val_y = train_test_split(
 list(zip(train_df['question_text'].values, features)) ,
train_df['target'].values,
test_size=0.2, random_state=2018 
)
 test_X = test_df["question_text"].values
train_X, features = zip(*train_X)
 val_X, val_features = zip(*valid_X)
 tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 return train_X, val_X, test_X, train_y, val_y, features, val_features, test_features, tokenizer.word_index
with strategy.scope() :
transformer_encoder = TFAutoModel.from_pretrained(model_name)
 input_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_ids")
 sequence_output = transformer_encoder(input_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(3, activation='softmax' )(cls_token)
 model = Model(inputs=input_ids, outputs=out)
 model.compile(
 Adam(lr=1e-5),
loss='sparse_categorical_crossentropy',
metrics=['accuracy']
)<train_model>
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.005838499,0.48782197
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.0053247833,0.49346462
embed_size = all_embs.shape[1]
print(emb_mean,emb_std,"para")
 nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
n_steps = len(x_train)// batch_size
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=n_epochs
)<train_model>
def model_cnn(embedding_matrix):
filter_sizes = [1,2,3,5]
num_filters = 36
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Reshape(( maxlen, embed_size, 1))(x)
 maxpool_pool = []
for i in range(len(filter_sizes)) :
conv = Conv2D(num_filters, kernel_size=(filter_sizes[i], embed_size),
kernel_initializer='he_normal', activation='elu' )(x)
 maxpool_pool.append(MaxPool2D(pool_size=(maxlen - filter_sizes[i] + 1, 1))(conv))
z = Concatenate(axis=1 )(maxpool_pool)
 z = Flatten()(z)
 z = Dropout(0.1 )(z)
 outp = Dense(1, activation="sigmoid" )(z)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=1
)<save_to_csv>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
test_preds = model.predict(test_dataset, verbose=1)
 submission['prediction'] = test_preds.argmax(axis=1)
 submission.to_csv("submission.csv" )<install_modules>
def model_lstm_atten(embedding_matrix):
inp1 = Input(shape=(maxlen,), name='inp1')
 inp2 = Input(shape=(nb_features,), name='inp2')
 x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp1)
 x = SpatialDropout1D(0.25 )(x)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = concatenate([x, inp2])
 x = Dense(128, activation="relu" )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=[inp1, inp2], outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
!pip install tfa-nightly<install_modules>
def train_pred(model, epochs=2):
for e in range(epochs):
model.fit({'inp1': train_X, 'inp2': features}, train_y, batch_size=512, epochs=1,
validation_data=({'inp1': val_X, 'inp2': val_features}, val_y))
pred_val_y = model.predict({'inp1': val_X, 'inp2': val_features}, batch_size=1024, verbose=0)
 pred_test_y = model.predict({'inp1': test_X, 'inp2': test_features}, batch_size=1024, verbose=0)
 return pred_val_y, pred_test_y
!pip install -q pyyaml h5py<import_modules>
train_X, val_X, test_X, train_y, val_y, features, val_features, test_features, word_index = load_and_prec()
import tensorflow as tf
from kaggle_datasets import KaggleDatasets
from matplotlib import pyplot as plt
import numpy as np
import random
import tensorflow_addons as tfa<define_variables>
np.save("train_X",train_X)
 np.save("val_X",val_X)
 np.save("test_X",test_X)
 np.save("train_y",train_y)
 np.save("val_y",val_y)
 np.save("features",features)
 np.save("val_features",val_features)
 np.save("test_features",test_features)
 np.save("word_index.npy",word_index)
 train_X = np.load("train_X.npy")
 val_X = np.load("val_X.npy")
 test_X = np.load("test_X.npy")
 train_y = np.load("train_y.npy")
 val_y = np.load("val_y.npy")
 features = np.load("features.npy")
 val_features = np.load("val_features.npy")
 test_features = np.load("test_features.npy")
 word_index = np.load("word_index.npy" ).item()
random.seed(1 )<define_variables>
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_2 = load_fasttext(word_index)
 embedding_matrix_3 = load_para(word_index )
GCS_DS_PATH = KaggleDatasets().get_gcs_path()<define_variables>
embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_2, embedding_matrix_3], axis = 0)
 np.shape(embedding_matrix )
AUTO = tf.data.experimental.AUTOTUNE
ignore_order = tf.data.Options()
ignore_order.experimental_deterministic = False
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64)}
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
IMAGE_SIZE = [512,512]
EPOCHS = 20
BATCH_SIZE = 16 * strategy.num_replicas_in_sync<define_variables>
outputs = []
training_data = tf.io.gfile.glob(GCS_DS_PATH + "/tfrecords-jpeg-512x512/train/*.tfrec")
 validation_data = tf.io.gfile.glob(GCS_DS_PATH + "/tfrecords-jpeg-512x512/val/*.tfrec")
 testing_data = tf.io.gfile.glob(GCS_DS_PATH + "/tfrecords-jpeg-512x512/test/*.tfrec")
 NUM_CLASSES = 104
NUM_TRAINING_IMAGES = 12753
NUM_TEST_IMAGES = 7382
STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE<normalization>
pred_val_y, pred_test_y = train_pred(model_lstm_atten(embedding_matrix), epochs = EPOCHS)
 outputs.append([pred_val_y, pred_test_y, '2 LSTM w/ attention'] )
def augment_data(image , label):
check = random.randint(0,9)
 if check == 1:
image = flip(image)
 elif check == 2:
image = color(image)
 elif check == 3:
image = rotate(image)
 elif check == 4:
image = shear(image)
 elif check == 5:
image = random_all(image)
 return image , label<categorify>
score = 0
thresh =.5
for i in np.arange(0.1, 0.991, 0.01):
y_val =(np.array(pred_val_y)> i ).astype(np.int)
 temp_score = f1_score(val_y, y_val)
 if(temp_score > score):
score = temp_score
thresh = i
print("CV: {}, Threshold: {}".format(score, thresh))
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image<categorify>
pred_test_y = np.mean([outputs[i][1] for i in range(len(outputs)) ], axis = 0)
 pred_test_y =(pred_test_y > thresh ).astype(int)
 test_df = pd.read_csv(".. /input/test.csv", usecols=["qid"])
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )
def read_labeled_tfrecord(record):
record = tf.io.parse_single_example(record , LABELED_TFREC_FORMAT)
 image = decode_image(record['image'])
 label = tf.cast(record['class'] , tf.int32)
 return image , label 
<split>
t_finish = time.time()
print(f"Kernel run time = {(t_finish-t_start)/3600} hours" )
<create_dataframe><EOS> 
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 _train = add_features(train_df)
 _test = add_features(test_df)
 features = _train.fillna(0)
 test_features = _test.fillna(0)
 ss = StandardScaler()
ss.fit(np.vstack(( features, test_features)))
 features = ss.transform(features)
 test_features = ss.transform(test_features)