Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<prepare_output>
pd.set_option("display.max_colwidth", 200)
 sns.set_style('darkgrid')
 pd.options.display.float_format = '{:,.3f}'.format
val_preds = np.average(checkpoint_val_preds, weights=weights, axis=0 )<compute_test_metric>
%%time
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print(train.shape, test.shape )
def power_mean(x, p=-5):
return np.power(np.mean(np.power(x, p)) ,1/p)
 def get_s_auc(y_true,y_pred,y_identity):
mask = y_identity==1
s_auc = roc_auc_score(y_true[mask],y_pred[mask])
 return s_auc
def get_bpsn_auc(y_true,y_pred,y_identity):
mask =(y_identity==1)&(y_true==0)|(y_identity==0)&(y_true==1)
 bpsn_auc = roc_auc_score(y_true[mask],y_pred[mask])
 return bpsn_auc
def get_bspn_auc(y_true,y_pred,y_identity):
mask =(y_identity==1)&(y_true==1)|(y_identity==0)&(y_true==0)
 bspn_auc = roc_auc_score(y_true[mask],y_pred[mask])
 return bspn_auc
def get_total_auc(y_true,y_pred,y_identities):
N = y_identities.shape[1]
saucs = np.array([get_s_auc(y_true,y_pred,y_identities[:,i])for i in range(N)])
 bpsns = np.array([get_bpsn_auc(y_true,y_pred,y_identities[:,i])for i in range(N)])
 bspns = np.array([get_bspn_auc(y_true,y_pred,y_identities[:,i])for i in range(N)])
 M_s_auc = power_mean(saucs)
 M_bpsns_auc = power_mean(bpsns)
 M_bspns_auc = power_mean(bspns)
 rauc = roc_auc_score(y_true,y_pred)
 total_auc = M_s_auc + M_bpsns_auc + M_bspns_auc + rauc
total_auc/= 4
return total_auc<compute_test_metric>
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )
get_total_auc(y_train[val_ind][:,0],val_preds,y_identities[val_ind] )<save_to_csv>
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }
ZEN = "".join(chr(0xff01 + i)for i in range(94))
HAN = "".join(chr(0x21 + i)for i in range(94))
ZEN2HAN = str.maketrans(ZEN, HAN)
 def preprocess(data):
def clean_special_chars(text):
punct = "/-'?!.,
for p in punct:
text = text.replace(p, ' ')
 for p in punct_mapping:
text = text.replace(p, punct_mapping[p])
 text = text.translate(ZEN2HAN)
 return text
data = data.astype(str ).apply(lambda x: clean_special_chars(x))
return data
train['question_text'] = preprocess(train['question_text'])
 test['question_text'] = preprocess(test['question_text'] )
df_submit = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv')
 df_submit.prediction = predictions
df_submit.to_csv('submission.csv', index=False )<install_modules>
text_to_word_sequence(train['question_text'].values[0] )
!pip install torch==1.6.0+cu101 torchvision==0.7.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html<import_modules>
X_train = train.drop(['qid','target'], axis=1)
 Y_train = train['target']
X_test = test.drop(['qid'], axis=1)
 del train, test
print(X_train.shape, X_test.shape )
torch.__version__<install_modules>
TARGET_COLUMN = 'target'
TEXT_COLUMN = 'question_text'
MAX_NUM_WORDS = 300000
TOKENIZER_FILTER = '\r\t
'
tokenizer = Tokenizer(num_words=MAX_NUM_WORDS, filters=TOKENIZER_FILTER)
 tokenizer.fit_on_texts(list(X_train[TEXT_COLUMN])+ list(X_test[TEXT_COLUMN]))
!pip install fastai==2.0.13<import_modules>
tokenizer_tx = Tokenizer(num_words=MAX_NUM_WORDS, filters=TOKENIZER_FILTER)
 tokenizer_tx.fit_on_texts(list(X_train.loc[Y_train == 1, TEXT_COLUMN]))
counter = sorted(dict(tokenizer_tx.word_docs ).items() , key=lambda x:x[1], reverse=True)
 wordcount_tx = pd.Series([x[1] for x in counter], [x[0] for x in counter])
 wordcount_stats = pd.concat([wordcount, wordcount_tx], axis=1, keys=['all', 'toxic'], sort=False)
 wordcount_tx = wordcount_stats[wordcount_stats['all'] * 0.8 <= wordcount_stats['toxic']].copy()
wordcount_tx[wordcount_tx['toxic'] > 0]
wordcount_tx.sort_values(by='toxic', ascending=False, inplace=True)
 print(len(wordcount_tx))
wordcount_tx['toxic'][:30].plot.bar(color='red', width=0.7, figsize=(12,3))
torch.__version__, fastai.__version__<import_modules>
wordcount = pd.concat([wordcount_tx['all'], wordcount] ).to_frame().reset_index()
wordcount.drop_duplicates(keep='first', inplace=True)
 wordcount = wordcount.set_index('index')[0]
del counter, wordcount_tx, wordcount_stats
import fastai
from fastai.text.all import *<load_from_csv>
VOCAB_SIZE = 50000
print('covered until', wordcount[VOCAB_SIZE], 'times word' )
train_df = pd.read_csv(path/'train.csv')
 train_df.head(5 )<load_from_csv>
%%time
EMBEDDINGS_DIMENSION = 300
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f)
 def build_matrix(path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( VOCAB_SIZE + 1, EMBEDDINGS_DIMENSION))
unknown_words = []
for i in range(VOCAB_SIZE):
try:
word = wordcount.index[i]
embedding_matrix[i] = embedding_index[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words
glove_matrix, unknown_words_glove = build_matrix(EMBEDDING_FILE)
 word2index = dict(( wordcount.index[i], i)for i in range(VOCAB_SIZE))
embedding_matrix = glove_matrix
embedding_matrix.shape
test_df = pd.read_csv(path/'test.csv')
 test_df.head(5 )<create_dataframe>
words_count = len(unknown_words_glove)
 print('n unknown words(glove):', words_count, ', {:.3%} of all words'.format(words_count / n_words))
print('unknown words(glove):', unknown_words_glove )
op = pd.concat([train_df['text'], test_df['text']])
 op = pd.DataFrame(op)
 len(op), len(train_df)+len(test_df )<create_dataframe>
MAX_SEQUENCE_LENGTH = 128
def word_index(word):
try:
return word2index[word]
except KeyError:
return VOCAB_SIZE
def pad_text(texts, tokenizer):
matrix = [list(map(word_index, text_to_word_sequence(t, filters=TOKENIZER_FILTER)))for t in texts]
return pad_sequences(matrix, maxlen=MAX_SEQUENCE_LENGTH)
 train_text = pad_text(X_train[TEXT_COLUMN], tokenizer)
 test_text = pad_text(X_test[TEXT_COLUMN], tokenizer )
dls_lm = TextDataLoaders.from_df(op, path=path, text_col='text', is_lm=True, valid_col=None)
 dls_lm.show_batch(max_n=3 )<load_pretrained>
del(X_train, X_test)
 gc.collect()
print(pd.DataFrame([[val for val in dir() ], [sys.getsizeof(eval(val)) for val in dir() ]],
index=['name','size'] ).T.sort_values('size', ascending=False ).reset_index(drop=True)[:10] )
learn = language_model_learner(dls_lm, AWD_LSTM, metrics=[accuracy, Perplexity() ], path='/kaggle/working/', wd=0.1 ).to_fp16()<set_options>
class F1Callback(Callback):
def __init__(self):
self.f1s = []
def on_epoch_end(self, epoch, logs):
eps = np.finfo(np.float32 ).eps
recall = logs["val_true_positives"] /(logs["val_possible_positives"] + eps)
 precision = logs["val_true_positives"] /(logs["val_predicted_positives"] + eps)
 f1 = 2*precision*recall /(precision+recall+eps)
 print("f1_val(from log)=", f1)
 self.f1s.append(f1)
 def true_positives(y_true, y_pred):
return K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 def possible_positives(y_true, y_pred):
return K.sum(K.round(K.clip(y_true, 0, 1)))
 def predicted_positives(y_true, y_pred):
return K.sum(K.round(K.clip(y_pred, 0, 1)) )
torch.cuda.is_available()<train_model>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
learn.fit_one_cycle(1, 1e-2 )<save_model>
def build_model(lr=0.0, lr_d=0.0, units=64, spatial_dr=0.0,
dense_units=0, dr=0.1, conv_size=32, epochs=20):
file_path = "best_model.hdf5"
check_point = ModelCheckpoint(file_path, monitor="val_loss", verbose=1, save_best_only=True, mode="min")
 early_stop = EarlyStopping(monitor="val_loss", mode="min", patience=3)
 sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
 embedding_layer = Embedding(*embedding_matrix.shape,
weights=[embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=True)
 x = embedding_layer(sequence_input)
 x = SpatialDropout1D(spatial_dr )(x)
 x = Bidirectional(CuDNNGRU(units, return_sequences=True))(x)
 x = Conv1D(conv_size, 2, padding="valid", kernel_initializer="he_uniform" )(x)
 avg_pool1 = GlobalAveragePooling1D()(x)
 max_pool1 = GlobalMaxPooling1D()(x)
 x = concatenate([avg_pool1, max_pool1])
 x = BatchNormalization()(x)
 x = Dense(int(dense_units / 2), activation='relu' )(x)
 x = Dropout(dr )(x)
 preds = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=sequence_input, outputs=preds)
 model.compile(loss="binary_crossentropy", optimizer=Adam(lr=lr, decay=lr_d),
metrics=['acc', true_positives, possible_positives, predicted_positives])
 model.summary()
history = model.fit(train_text, Y_train, batch_size=1024, epochs=epochs, validation_split=0.1,
verbose=1, callbacks=[check_point, early_stop, F1Callback() ])
 return model
learn.save('1epoch' )<load_pretrained>
model = build_model(lr=1e-3, lr_d=1e-7, units=64, spatial_dr=0.2, dense_units=64, dr=0.1, conv_size=64, epochs=10)
 pred = model.predict(test_text, batch_size=1024 )
learn = learn.load('1epoch' )<train_model>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
learn.unfreeze()
learn.fit_one_cycle(10, 1e-3 )<categorify>
pred =(pred[:,0] > 0.5 ).astype(np.int)
 submission = pd.read_csv('.. /input/sample_submission.csv', index_col='qid')
 submission['prediction'] = pred
submission.reset_index(drop=False, inplace=True)
 submission.to_csv('submission.csv', index=False)
 submission.head()
learn.save_encoder('finetuned' )<load_from_csv>
pd.Series(pred ).value_counts().to_frame().T / len(pred )