Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<filter>
embed_size = 300
max_features = 100000
maxlen = 80
train.loc[train['SentenceId'] == 1]<string_transform>
tqdm.pandas()
def clean_sentences(df):
reviews = []
for sent in tqdm(df['Phrase']):
review_text = BeautifulSoup(sent ).get_text()
review_text = re.sub("[^a-zA-Z]"," ", review_text)
 words = word_tokenize(review_text.lower())
 lemma_words = [lemmatizer.lemmatize(i)for i in words]
reviews.append(lemma_words)
 return(reviews )<string_transform>
def check_coverage(vocab,embeddings_index,word_counts):
a = {}
oov = {}
k = 0
i = 0
for word in tqdm(vocab):
try:
a[word] = embeddings_index[word]
k += word_counts[word]
except:
oov[word] = word_counts[word]
i += word_counts[word]
print('Found embeddings for {:.2%} of vocab'.format(len(a)/ len(vocab)))
 print('Found embeddings for {:.2%} of all text'.format(k /(k + i)))
 sorted_x = sorted(oov.items() , key=operator.itemgetter(1)) [::-1]
return sorted_x
train_sentences = clean_sentences(train)
 test_sentences = clean_sentences(test)
 print(len(train_sentences))
print(len(test_sentences))<categorify>
def load_glove(word_index, max_features):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.005838499,0.48782197
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix, embeddings_index
target=train.Sentiment.values
y_target=to_categorical(target)
 num_classes=y_target.shape[1] 
<split>
def load_para(word_index, max_features):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix, embeddings_index
X_train,X_val,y_train,y_val = train_test_split(train_sentences,y_target,
test_size=0.2,stratify=y_target )<count_unique_values>
def fit_for_glove(X):
contraction_map = {"ain't": "is not", "aren't": "are not","can't": "cannot",
"can't've": "cannot have", "'cause": "because", "could've": "could have",
"couldn't": "could not", "couldn't've": "could not have","didn't": "did not",
"doesn't": "does not", "don't": "do not", "hadn't": "had not",
"hadn't've": "had not have", "hasn't": "has not", "haven't": "have not",
"he'd": "he would", "he'd've": "he would have", "he'll": "he will",
"he'll've": "he he will have", "he's": "he is", "how'd": "how did",
"how'd'y": "how do you", "how'll": "how will", "how's": "how is",
"I'd": "I would", "I'd've": "I would have", "I'll": "I will",
"I'll've": "I will have","I'm": "I am", "I've": "I have",
"i'd": "i would", "i'd've": "i would have", "i'll": "i will",
"i'll've": "i will have","i'm": "i am", "i've": "i have",
"isn't": "is not", "it'd": "it would", "it'd've": "it would have",
"it'll": "it will", "it'll've": "it will have","it's": "it is",
"let's": "let us", "ma'am": "madam", "mayn't": "may not",
"might've": "might have","mightn't": "might not","mightn't've": "might not have",
"must've": "must have", "mustn't": "must not", "mustn't've": "must not have",
"needn't": "need not", "needn't've": "need not have","o'clock": "of the clock",
"oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not",
"sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would",
"she'd've": "she would have", "she'll": "she will", "she'll've": "she will have",
"she's": "she is", "should've": "should have", "shouldn't": "should not",
"shouldn't've": "should not have", "so've": "so have","so's": "so as",
"this's": "this is",
"that'd": "that would", "that'd've": "that would have","that's": "that is",
"there'd": "there would", "there'd've": "there would have","there's": "there is",
"they'd": "they would", "they'd've": "they would have", "they'll": "they will",
"they'll've": "they will have", "they're": "they are", "they've": "they have",
"to've": "to have", "wasn't": "was not", "we'd": "we would",
"we'd've": "we would have", "we'll": "we will", "we'll've": "we will have",
"we're": "we are", "we've": "we have", "weren't": "were not",
"what'll": "what will", "what'll've": "what will have", "what're": "what are",
"what's": "what is", "what've": "what have", "when's": "when is",
"when've": "when have", "where'd": "where did", "where's": "where is",
"where've": "where have", "who'll": "who will", "who'll've": "who will have",
"who's": "who is", "who've": "who have", "why's": "why is",
"why've": "why have", "will've": "will have", "won't": "will not",
"won't've": "will not have", "would've": "would have", "wouldn't": "would not",
"wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would",
"y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have",
"you'd": "you would", "you'd've": "you would have", "you'll": "you will",
"you'll've": "you will have", "you're": "you are", "you've": "you have"}
cap = {}
for k, v in contraction_map.items() :
cap[k.capitalize() ] = v.capitalize()
contraction_map.update(cap)
 c_re = re.compile('(%s)' % '|'.join(contraction_map.keys()))
def expandContractions(text, c_re=c_re):
def replace_m(match):
return contraction_map[match.group(0)]
t = text.replace('’', "'")
 t = t.replace('“', "")
 punct = "/-'?!.,
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi'}
def clean_special_chars(t, punct, mapping):
for p in mapping:
t = t.replace(p, mapping[p])
 for p in punct:
t = t.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
t = t.replace(s, specials[s])
 return t
t = clean_special_chars(t, punct, punct_mapping)
 mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
t = correct_spelling(t, mispell_dict)
 return c_re.sub(replace_m, t.lower())
 return X.progress_apply(lambda x: expandContractions(x))
unique_words = set()
len_max = 0
for sent in tqdm(X_train):
unique_words.update(sent)
 if(len_max<len(sent)) :
len_max = len(sent)
 print(len(list(unique_words)))
 print(len_max )<string_transform>
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
 def f1(y_true, y_pred):

 def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
tokenizer = Tokenizer(num_words=len(list(unique_words)))
 tokenizer.fit_on_texts(list(X_train))
X_train = tokenizer.texts_to_sequences(X_train)
 X_val = tokenizer.texts_to_sequences(X_val)
 X_test = tokenizer.texts_to_sequences(test_sentences)
 X_train = sequence.pad_sequences(X_train, maxlen=len_max)
 X_val = sequence.pad_sequences(X_val, maxlen=len_max)
 X_test = sequence.pad_sequences(X_test, maxlen=len_max)
 print(X_train.shape,X_val.shape,X_test.shape )<choose_model_class>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
score = metrics.f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
early_stopping = EarlyStopping(min_delta = 0.001, mode = 'max', monitor='val_acc', patience = 2)
 callback = [early_stopping]<choose_model_class>
def train_pred(model, train_X, train_y, val_X, val_y, epochs=2, callback=None):
for e in range(epochs):
model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y), callbacks = callback, verbose=0)
 pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)
 best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
if score > best_score:
best_thresh = thresh
best_score = score
print("Epoch: ", e, "- Val F1 Score: {:.4f}".format(best_score))
pred_test_y = model.predict([X_real], batch_size=1024, verbose=0)
 print('='*100)
 return pred_val_y, pred_test_y, best_score
model=Sequential()
model.add(Embedding(len(list(unique_words)) ,300,input_length=len_max))
model.add(LSTM(128,dropout=0.5, recurrent_dropout=0.5,return_sequences=True))
model.add(LSTM(64,dropout=0.5, recurrent_dropout=0.5,return_sequences=False))
model.add(Dense(100,activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes,activation='softmax'))
model.compile(loss='categorical_crossentropy',optimizer=Adam(lr=0.005),metrics=['accuracy'])
 model.summary()<train_model>
df = pd.read_csv('.. /input/train.csv' )
history=model.fit(X_train, y_train, validation_data=(X_val, y_val),
epochs=6, batch_size=256, verbose=1, callbacks=callback )<save_to_csv>
X, y = df['question_text'], df['target'].values
y_pred=model.predict_classes(X_test)
 sub_file = pd.read_csv('.. /input/sentiment-analysis-on-movie-reviews/sampleSubmission.csv',sep=',')
 sub_file.Sentiment=y_pred
sub_file.to_csv('Submission.csv',index=False )<set_options>
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(X)
 X_seq = tokenizer.texts_to_sequences(X)
 X_seq = pad_sequences(X_seq, maxlen=maxlen )
def seed_all(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_all(2020 )<install_modules>
emb_mat_1, emb_idx_1 = load_glove(tokenizer.word_index, max_features )
!pip install transformers<load_pretrained>
emb_mat_2, emb_idx_2 = load_para(tokenizer.word_index, max_features )
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased' )<string_transform>
emb_mat = np.mean([emb_mat_1, emb_mat_2], axis = 0 )
def tokenize_and_cut(sentence):
tokens = tokenizer.tokenize(sentence)
 max_input_length = tokenizer.max_model_input_sizes['bert-base-uncased']
tokens = tokens[:max_input_length-2]
return tokens<categorify>
X_train, y_train = X_seq, y
TEXT = data.Field(batch_first = True,
use_vocab = False,
tokenize = tokenize_and_cut,
preprocessing = tokenizer.convert_tokens_to_ids,
init_token = tokenizer.cls_token_id,
eos_token = tokenizer.sep_token_id,
pad_token = tokenizer.pad_token_id,
unk_token = tokenizer.unk_token_id)
 LABEL = data.Field(sequential=False, use_vocab=False )<load_pretrained>
test_df = pd.read_csv('.. /input/test.csv')
 X_real = test_df['question_text']
!mkdir./data
!unzip -d./data.. /input/sentiment-analysis-on-movie-reviews/train.tsv.zip
!unzip -d./data.. /input/sentiment-analysis-on-movie-reviews/test.tsv.zip<save_to_csv>
X_real = tokenizer.texts_to_sequences(X_real)
 X_real = pad_sequences(X_real, maxlen=maxlen )
train = pd.read_csv('./data/train.tsv', sep='\t')
 test = pd.read_csv('./data/test.tsv', sep='\t')
 train, valid = train_test_split(train, test_size=0.2)
 train.to_csv('./data/train.csv', index=False)
 valid.to_csv('./data/validation.csv', index=False)
 train, valid = data.TabularDataset.splits(
 path='./data', train='train.csv', validation='validation.csv', format='csv', skip_header=True,
fields=[('PhraseId', None),('SentenceId', None),('Phrase', TEXT),('Sentiment', LABEL)])
 test = data.TabularDataset('./data/test.tsv', format='tsv', skip_header=True,
fields=[('PhraseId', None),('SentenceId', None),('Phrase', TEXT)] )<train_model>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
print(f"Number of training examples: {len(train)}")
 print(f"Number of validation examples: {len(valid)}")
 print(f"Number of testing examples: {len(test)}" )<data_type_conversions>
def model_lstm_atten_bi(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(40, return_sequences=True))(x)
 y = Bidirectional(CuDNNLSTM(40, return_sequences=True))(x)
 atten_1 = Attention(maxlen )(x)
 atten_2 = Attention(maxlen )(y)
 avg_pool = GlobalAveragePooling1D()(y)
 max_pool = GlobalMaxPooling1D()(y)
 conc = concatenate([atten_1, atten_2, avg_pool, max_pool])
 conc = Dense(32, activation="relu" )(conc)
 conc = Dropout(0.1 )(conc)
 outp = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=[f1])
 return model
print(tokenizer.convert_ids_to_tokens(vars(train[6])['Phrase']))<define_variables>
def model_lstm_cnn(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix],trainable = False )(inp)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(50, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(50, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(32, return_sequences=True))(x)
 x = Conv1D(32, kernel_size = 3, padding = "valid", kernel_initializer = "glorot_uniform" )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 x = concatenate([avg_pool, max_pool])
 x = Dense(32, activation='relu' )(x)
 x = Dropout(0.1 )(x)
 outp = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy',optimizer='rmsprop',metrics=[f1])
 return model
BATCH_SIZE = 256
DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 train_iter = data.BucketIterator(train, batch_size=BATCH_SIZE, shuffle=True, device=DEVICE)
 valid_iter = data.BucketIterator(valid, batch_size=BATCH_SIZE, shuffle=True, device=DEVICE)
 test_iter = data.Iterator(test, batch_size=BATCH_SIZE, train=False, sort=False, device=DEVICE )<load_pretrained>
clr = CyclicLR(base_lr=0.001, max_lr=0.004,
step_size=300., mode='exp_range',
gamma=0.99994)
 train_meta = np.zeros(y_train.shape)
 test_meta = np.zeros(X_real.shape[0])
 splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=14 ).split(X_train, y_train))
for idx,(train_idx, valid_idx)in enumerate(splits):
X_train1 = X_train[train_idx]
y_train1 = y_train[train_idx]
X_val = X_train[valid_idx]
y_val = y_train[valid_idx]
model = model_lstm_atten_bi(emb_mat)
 pred_val_y, pred_test_y, best_score = train_pred(model, X_train1, y_train1, X_val, y_val, epochs = 7, callback = [clr,])
 train_meta[valid_idx] = pred_val_y.reshape(-1)
 test_meta += pred_test_y.reshape(-1)/ len(splits )
<choose_model_class><EOS> 
search_result = threshold_search(y_train, train_meta)
 print(search_result)
 sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.prediction = test_meta > search_result['threshold']
sub.to_csv("submission.csv", index=False )