Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<categorify>
tqdm.pandas()
target = train.Sentiment.values
y_target = to_categorical(target)
 num_classes = y_target.shape[1]<split>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print("Train shape : ", train.shape)
 print("Test shape : ", test.shape )
X_train, X_val, y_train, y_val = train_test_split(train_sentences, y_target, test_size=0.2, stratify=y_target )<count_unique_values>
train["question_text"] = train["question_text"].str.lower()
test["question_text"] = test["question_text"].str.lower()
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
train["question_text"] = train["question_text"].apply(lambda x: clean_text(x))
test["question_text"] = test["question_text"].apply(lambda x: clean_text(x))
unique_words = set()
len_max = 0
for sent in tqdm(X_train):
unique_words.update(sent)
 if len_max < len(sent):
len_max = len(sent)
 len(list(unique_words)) , len_max<string_transform>
embed_size = 300
max_features = None
maxlen = 72
X = train["question_text"].fillna("_na_" ).values
X_test = test["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(X))
X = tokenizer.texts_to_sequences(X)
 X_test = tokenizer.texts_to_sequences(X_test)
 X = pad_sequences(X, maxlen=maxlen)
 X_test = pad_sequences(X_test, maxlen=maxlen)
 Y = train['target'].values
sub = test[['qid']]
tokenizer = Tokenizer(num_words=len(list(unique_words)))
 tokenizer.fit_on_texts(list(X_train))
X_train = tokenizer.texts_to_sequences(X_train)
 X_val = tokenizer.texts_to_sequences(X_val)
 X_test = tokenizer.texts_to_sequences(test_sentences)
 X_train = sequence.pad_sequences(X_train, maxlen=len_max)
 X_val = sequence.pad_sequences(X_val, maxlen=len_max)
 X_test = sequence.pad_sequences(X_test, maxlen=len_max)
 X_train.shape, X_val.shape, X_test.shape<import_modules>
del train, test
gc.collect()
from keras.callbacks import EarlyStopping
from keras.models import Sequential
from keras.layers import Embedding, LSTM, Dense, Dropout
from keras.optimizers import Adam<choose_model_class>
word_index = tokenizer.word_index
max_features = len(word_index)+1
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100 and o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100 and o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
early_stopping = EarlyStopping(min_delta=0.001, mode='max', monitor='val_acc', patience=2)
 callback = [early_stopping]<choose_model_class>
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_3 = load_para(word_index)
 embedding_matrix = np.mean(( embedding_matrix_1, embedding_matrix_3), axis=0)
 del embedding_matrix_1, embedding_matrix_3
gc.collect()
np.shape(embedding_matrix )
model = Sequential()
model.add(Embedding(len(list(unique_words)) , 300, input_length=len_max))
model.add(LSTM(128, dropout=0.5, recurrent_dropout=0.5, return_sequences=True))
model.add(LSTM(64, dropout=0.5, recurrent_dropout=0.5, return_sequences=False))
model.add(Dense(100, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.005), metrics=['accuracy'])
 model.summary()<train_model>
class CRF(Layer):

 def __init__(self, units,
learn_mode='join',
test_mode=None,
sparse_target=False,
use_boundary=True,
use_bias=True,
activation='linear',
kernel_initializer='glorot_uniform',
chain_initializer='orthogonal',
bias_initializer='zeros',
boundary_initializer='zeros',
kernel_regularizer=None,
chain_regularizer=None,
boundary_regularizer=None,
bias_regularizer=None,
kernel_constraint=None,
chain_constraint=None,
boundary_constraint=None,
bias_constraint=None,
input_dim=None,
unroll=False,
**kwargs):
super(CRF, self ).__init__(**kwargs)
 self.supports_masking = True
self.units = units
self.learn_mode = learn_mode
assert self.learn_mode in ['join', 'marginal']
self.test_mode = test_mode
if self.test_mode is None:
self.test_mode = 'viterbi' if self.learn_mode == 'join' else 'marginal'
else:
assert self.test_mode in ['viterbi', 'marginal']
self.sparse_target = sparse_target
self.use_boundary = use_boundary
self.use_bias = use_bias
self.activation = activations.get(activation)
 self.kernel_initializer = initializers.get(kernel_initializer)
 self.chain_initializer = initializers.get(chain_initializer)
 self.boundary_initializer = initializers.get(boundary_initializer)
 self.bias_initializer = initializers.get(bias_initializer)
 self.kernel_regularizer = regularizers.get(kernel_regularizer)
 self.chain_regularizer = regularizers.get(chain_regularizer)
 self.boundary_regularizer = regularizers.get(boundary_regularizer)
 self.bias_regularizer = regularizers.get(bias_regularizer)
 self.kernel_constraint = constraints.get(kernel_constraint)
 self.chain_constraint = constraints.get(chain_constraint)
 self.boundary_constraint = constraints.get(boundary_constraint)
 self.bias_constraint = constraints.get(bias_constraint)
 self.unroll = unroll
def build(self, input_shape):
self.input_spec = [InputSpec(shape=input_shape)]
self.input_dim = input_shape[-1]
self.kernel = self.add_weight(( self.input_dim, self.units),
name='kernel',
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
 self.chain_kernel = self.add_weight(( self.units, self.units),
name='chain_kernel',
initializer=self.chain_initializer,
regularizer=self.chain_regularizer,
constraint=self.chain_constraint)
 if self.use_bias:
self.bias = self.add_weight(( self.units,),
name='bias',
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
 else:
self.bias = None
if self.use_boundary:
self.left_boundary = self.add_weight(( self.units,),
name='left_boundary',
initializer=self.boundary_initializer,
regularizer=self.boundary_regularizer,
constraint=self.boundary_constraint)
 self.right_boundary = self.add_weight(( self.units,),
name='right_boundary',
initializer=self.boundary_initializer,
regularizer=self.boundary_regularizer,
constraint=self.boundary_constraint)
 self.built = True
def call(self, X, mask=None):
if mask is not None:
assert K.ndim(mask)== 2, 'Input mask to CRF must have dim 2 if not None'
if self.test_mode == 'viterbi':
test_output = self.viterbi_decoding(X, mask)
 else:
test_output = self.get_marginal_prob(X, mask)
 self.uses_learning_phase = True
if self.learn_mode == 'join':
train_output = K.zeros_like(K.dot(X, self.kernel))
out = K.in_train_phase(train_output, test_output)
 else:
if self.test_mode == 'viterbi':
train_output = self.get_marginal_prob(X, mask)
 out = K.in_train_phase(train_output, test_output)
 else:
out = test_output
return out
def compute_output_shape(self, input_shape):
return input_shape[:2] +(self.units,)
 def compute_mask(self, input, mask=None):
if mask is not None and self.learn_mode == 'join':
return K.any(mask, axis=1)
 return mask
def get_config(self):
config = {'units': self.units,
'learn_mode': self.learn_mode,
'test_mode': self.test_mode,
'use_boundary': self.use_boundary,
'use_bias': self.use_bias,
'sparse_target': self.sparse_target,
'kernel_initializer': initializers.serialize(self.kernel_initializer),
'chain_initializer': initializers.serialize(self.chain_initializer),
'boundary_initializer': initializers.serialize(self.boundary_initializer),
'bias_initializer': initializers.serialize(self.bias_initializer),
'activation': activations.serialize(self.activation),
'kernel_regularizer': regularizers.serialize(self.kernel_regularizer),
'chain_regularizer': regularizers.serialize(self.chain_regularizer),
'boundary_regularizer': regularizers.serialize(self.boundary_regularizer),
'bias_regularizer': regularizers.serialize(self.bias_regularizer),
'kernel_constraint': constraints.serialize(self.kernel_constraint),
'chain_constraint': constraints.serialize(self.chain_constraint),
'boundary_constraint': constraints.serialize(self.boundary_constraint),
'bias_constraint': constraints.serialize(self.bias_constraint),
'input_dim': self.input_dim,
'unroll': self.unroll}
base_config = super(CRF, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
@property
def loss_function(self):
if self.learn_mode == 'join':
def loss(y_true, y_pred):
assert self._inbound_nodes, 'CRF has not connected to any layer.'
assert not self._outbound_nodes, 'When learn_model="join", CRF must be the last layer.'
if self.sparse_target:
y_true = K.one_hot(K.cast(y_true[:, :, 0], 'int32'), self.units)
 X = self._inbound_nodes[0].input_tensors[0]
mask = self._inbound_nodes[0].input_masks[0]
nloglik = self.get_negative_log_likelihood(y_true, X, mask)
 return nloglik
return loss
else:
if self.sparse_target:
return sparse_categorical_crossentropy
else:
return categorical_crossentropy
@property
def accuracy(self):
if self.test_mode == 'viterbi':
return self.viterbi_acc
else:
return self.marginal_acc
@staticmethod
def _get_accuracy(y_true, y_pred, mask, sparse_target=False):
y_pred = K.argmax(y_pred, -1)
 if sparse_target:
y_true = K.cast(y_true[:, :, 0], K.dtype(y_pred))
else:
y_true = K.argmax(y_true, -1)
 judge = K.cast(K.equal(y_pred, y_true), K.floatx())
 if mask is None:
return K.mean(judge)
 else:
mask = K.cast(mask, K.floatx())
 return K.sum(judge * mask)/ K.sum(mask)
 @property
def viterbi_acc(self):
def acc(y_true, y_pred):
X = self._inbound_nodes[0].input_tensors[0]
mask = self._inbound_nodes[0].input_masks[0]
y_pred = self.viterbi_decoding(X, mask)
 return self._get_accuracy(y_true, y_pred, mask, self.sparse_target)
 acc.func_name = 'viterbi_acc'
return acc
@property
def marginal_acc(self):
def acc(y_true, y_pred):
X = self._inbound_nodes[0].input_tensors[0]
mask = self._inbound_nodes[0].input_masks[0]
y_pred = self.get_marginal_prob(X, mask)
 return self._get_accuracy(y_true, y_pred, mask, self.sparse_target)
 acc.func_name = 'marginal_acc'
return acc
@staticmethod
def softmaxNd(x, axis=-1):
m = K.max(x, axis=axis, keepdims=True)
 exp_x = K.exp(x - m)
 prob_x = exp_x / K.sum(exp_x, axis=axis, keepdims=True)
 return prob_x
@staticmethod
def shift_left(x, offset=1):
assert offset > 0
return K.concatenate([x[:, offset:], K.zeros_like(x[:, :offset])], axis=1)
 @staticmethod
def shift_right(x, offset=1):
assert offset > 0
return K.concatenate([K.zeros_like(x[:, :offset]), x[:, :-offset]], axis=1)
 def add_boundary_energy(self, energy, mask, start, end):
start = K.expand_dims(K.expand_dims(start, 0), 0)
 end = K.expand_dims(K.expand_dims(end, 0), 0)
 if mask is None:
energy = K.concatenate([energy[:, :1, :] + start, energy[:, 1:, :]], axis=1)
 energy = K.concatenate([energy[:, :-1, :], energy[:, -1:, :] + end], axis=1)
 else:
mask = K.expand_dims(K.cast(mask, K.floatx()))
start_mask = K.cast(K.greater(mask, self.shift_right(mask)) , K.floatx())
 end_mask = K.cast(K.greater(self.shift_left(mask), mask), K.floatx())
 energy = energy + start_mask * start
energy = energy + end_mask * end
return energy
def get_log_normalization_constant(self, input_energy, mask, **kwargs):

 logZ = self.recursion(input_energy, mask, return_sequences=False, **kwargs)
 return logZ[:, 0]
def get_energy(self, y_true, input_energy, mask):

 input_energy = K.sum(input_energy * y_true, 2)
 chain_energy = K.sum(K.dot(y_true[:, :-1, :], self.chain_kernel)* y_true[:, 1:, :], 2)
 if mask is not None:
mask = K.cast(mask, K.floatx())
 chain_mask = mask[:, :-1] * mask[:, 1:]
input_energy = input_energy * mask
chain_energy = chain_energy * chain_mask
total_energy = K.sum(input_energy, -1)+ K.sum(chain_energy, -1)
 return total_energy
def get_negative_log_likelihood(self, y_true, X, mask):

 input_energy = self.activation(K.dot(X, self.kernel)+ self.bias)
 if self.use_boundary:
input_energy = self.add_boundary_energy(input_energy, mask, self.left_boundary, self.right_boundary)
 energy = self.get_energy(y_true, input_energy, mask)
 logZ = self.get_log_normalization_constant(input_energy, mask, input_length=K.int_shape(X)[1])
 nloglik = logZ + energy
if mask is not None:
nloglik = nloglik / K.sum(K.cast(mask, K.floatx()), 1)
 else:
nloglik = nloglik / K.cast(K.shape(X)[1], K.floatx())
 return nloglik
def step(self, input_energy_t, states, return_logZ=True):
prev_target_val, i, chain_energy = states[:3]
t = K.cast(i[0, 0], dtype='int32')
 if len(states)> 3:
if K.backend() == 'theano':
m = states[3][:, t:(t + 2)]
else:
m = K.tf.slice(states[3], [0, t], [-1, 2])
 input_energy_t = input_energy_t * K.expand_dims(m[:, 0])
 chain_energy = chain_energy * K.expand_dims(K.expand_dims(m[:, 0] * m[:, 1]))
if return_logZ:
energy = chain_energy + K.expand_dims(input_energy_t - prev_target_val, 2)
 new_target_val = K.logsumexp(-energy, 1)
 return new_target_val, [new_target_val, i + 1]
else:
energy = chain_energy + K.expand_dims(input_energy_t + prev_target_val, 2)
 min_energy = K.min(energy, 1)
 argmin_table = K.cast(K.argmin(energy, 1), K.floatx())
 return argmin_table, [min_energy, i + 1]
def recursion(self, input_energy, mask=None, go_backwards=False, return_sequences=True, return_logZ=True, input_length=None):

 chain_energy = self.chain_kernel
chain_energy = K.expand_dims(chain_energy, 0)
 prev_target_val = K.zeros_like(input_energy[:, 0, :])
 if go_backwards:
input_energy = K.reverse(input_energy, 1)
 if mask is not None:
mask = K.reverse(mask, 1)
 initial_states = [prev_target_val, K.zeros_like(prev_target_val[:, :1])]
constants = [chain_energy]
if mask is not None:
mask2 = K.cast(K.concatenate([mask, K.zeros_like(mask[:, :1])], axis=1), K.floatx())
 constants.append(mask2)
 def _step(input_energy_i, states):
return self.step(input_energy_i, states, return_logZ)
 target_val_last, target_val_seq, _ = K.rnn(_step, input_energy, initial_states, constants=constants,
input_length=input_length, unroll=self.unroll)
 if return_sequences:
if go_backwards:
target_val_seq = K.reverse(target_val_seq, 1)
 return target_val_seq
else:
return target_val_last
def forward_recursion(self, input_energy, **kwargs):
return self.recursion(input_energy, **kwargs)
 def backward_recursion(self, input_energy, **kwargs):
return self.recursion(input_energy, go_backwards=True, **kwargs)
 def get_marginal_prob(self, X, mask=None):
input_energy = self.activation(K.dot(X, self.kernel)+ self.bias)
 if self.use_boundary:
input_energy = self.add_boundary_energy(input_energy, mask, self.left_boundary, self.right_boundary)
 input_length = K.int_shape(X)[1]
alpha = self.forward_recursion(input_energy, mask=mask, input_length=input_length)
 beta = self.backward_recursion(input_energy, mask=mask, input_length=input_length)
 if mask is not None:
input_energy = input_energy * K.expand_dims(K.cast(mask, K.floatx()))
margin = -(self.shift_right(alpha)+ input_energy + self.shift_left(beta))
return self.softmaxNd(margin)
 def viterbi_decoding(self, X, mask=None):
input_energy = self.activation(K.dot(X, self.kernel)+ self.bias)
 if self.use_boundary:
input_energy = self.add_boundary_energy(input_energy, mask, self.left_boundary, self.right_boundary)
 argmin_tables = self.recursion(input_energy, mask, return_logZ=False)
 argmin_tables = K.cast(argmin_tables, 'int32')
 argmin_tables = K.reverse(argmin_tables, 1)
 initial_best_idx = [K.expand_dims(argmin_tables[:, 0, 0])]
if K.backend() == 'theano':
initial_best_idx = [K.T.unbroadcast(initial_best_idx[0], 1)]
def gather_each_row(params, indices):
n = K.shape(indices)[0]
if K.backend() == 'theano':
return params[K.T.arange(n), indices]
else:
indices = K.transpose(K.stack([K.tf.range(n), indices]))
return K.tf.gather_nd(params, indices)
 def find_path(argmin_table, best_idx):
next_best_idx = gather_each_row(argmin_table, best_idx[0][:, 0])
 next_best_idx = K.expand_dims(next_best_idx)
 if K.backend() == 'theano':
next_best_idx = K.T.unbroadcast(next_best_idx, 1)
 return next_best_idx, [next_best_idx]
_, best_paths, _ = K.rnn(find_path, argmin_tables, initial_best_idx, input_length=K.int_shape(X)[1], unroll=self.unroll)
 best_paths = K.reverse(best_paths, 1)
 best_paths = K.squeeze(best_paths, 2)
 return K.one_hot(best_paths, self.units )
%%time
history = model.fit(X_train, y_train, validation_data=(X_val, y_val), epochs=10, batch_size=256, verbose=1, callbacks=callback )<save_to_csv>
def gru_crf(inp):
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(rate=0.24 )(x)
 x = Bidirectional(CuDNNGRU(100, return_sequences=True,
kernel_initializer=glorot_normal(seed=123000), recurrent_initializer=orthogonal(gain=1.0, seed=10000)) )(x)
 crf = CRF(10,learn_mode='marginal',unroll=True)
 x = crf(x)
 x = Flatten()(x)
 x = Dense(100, activation='relu', kernel_initializer=glorot_normal(seed=123000))(x)
 x = Dropout(0.2 )(x)
 x = BatchNormalization()(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam() ,)
 return model
y_pred = model.predict_classes(X_test)
 submission = pd.read_csv('/kaggle/input/sentiment-analysis-on-movie-reviews/sampleSubmission.csv')
 submission.Sentiment = y_pred
submission.to_csv('submission.csv', index=False )<import_modules>
def f1_smart(y_true, y_pred):
args = np.argsort(y_pred)
 tp = y_true.sum()
fs =(tp - np.cumsum(y_true[args[:-1]])) / np.arange(y_true.shape[0] + tp - 1, tp, -1)
 res_idx = np.argmax(fs)
 return 2 * fs[res_idx],(y_pred[args[res_idx]] + y_pred[args[res_idx + 1]])/ 2
lemmatizer = WordNetLemmatizer() 
<load_from_disk>
f1, threshold = f1_smart(np.squeeze(Y), oof)
 print('Optimal F1: {:.4f} at threshold: {:.4f}'.format(f1, threshold))
!unzip.. /input/sentiment-analysis-on-movie-reviews/train.tsv.zip<load_from_disk>
np.mean(bestscore), np.mean(logloss )
<load_from_csv><EOS> 
y_test = y_test.reshape(( -1, 1))
pred_test_y =(y_test>threshold ).astype(int)
 sub['prediction'] = pred_test_y
sub.to_csv("submission.csv", index=False )