Kaggle competition: Texts classification
<SOS> metric: categorizationaccuracy Kaggle data source: texts-classification<feature_engineering>
gc.enable()
train["question_text"] = train["question_text"].str.lower()
test["question_text"] = test["question_text"].str.lower()
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
train["question_text"] = train["question_text"].apply(lambda x: clean_text(x))
test["question_text"] = test["question_text"].apply(lambda x: clean_text(x))<prepare_x_and_y>
tf.compat.v1.disable_eager_execution()
embed_size = 300
max_features = None
maxlen = 72
X = train["question_text"].fillna("_na_" ).values
X_test = test["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features, filters='')
 tokenizer.fit_on_texts(list(X))
X = tokenizer.texts_to_sequences(X)
 X_test = tokenizer.texts_to_sequences(X_test)
 X = pad_sequences(X, maxlen=maxlen)
 X_test = pad_sequences(X_test, maxlen=maxlen)
 Y = train['target'].values
sub = test[['qid']]<set_options>
CATEGORY_CNT = 54
BATCH_SIZE = 256
PREDICT_BATCH_SIZE = 256
EPOCHS = 10
STEPS_PER_EPOCH = 3000
VAL_STEPS_PER_EPOCH = 100
WORD2VEC_DIM = 200
FASTTEXT_DIM = 200
EXP_DECAY_COEF = 0.75
VERBOSE = 1
MAXLEN = 300
TRAINABLE_EMBED_SIZE = 75
del train, test
gc.collect()<statistical_test>
def save_object(obj, name):
f = open(name, 'wb')
 pickle.dump(obj, f)
 f.close()
def load_object(name):
f = open(name, 'rb')
 obj = pickle.load(f)
 f.close()
return obj
word_index = tokenizer.word_index
max_features = len(word_index)+1
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100 and o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100 and o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<normalization>
b_cat_map = load_object(".. /input/bonus-task-pickled-data-1/b_cat_map.pkl")
 f_cat_map = load_object(".. /input/bonus-task-pickled-data-1/f_cat_map.pkl")
 embed_fasttext = load_object(".. /input/bonus-task-pickled-data-1/embed_fasttext.pkl")
 embed_word2vec = load_object(".. /input/bonus-task-pickled-data-1/embed_word2vec.pkl")
 itemid = load_object(".. /input/bonus-task-pickled-data-1/itemid.pkl")
 y_train = load_object(".. /input/bonus-task-pickled-data-1/y_train.pkl")
 word_index = load_object(".. /input/bonus-x/word_index.pkl" )
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_3 = load_para(word_index)
 embedding_matrix = np.mean(( embedding_matrix_1, embedding_matrix_3), axis=0)
 np.shape(embedding_matrix )<compute_test_metric>
def build_emb_matrix(word_index, embeddings_index, dim):
embedding_matrix = np.zeros(( len(word_index)+ 1, dim + 1), dtype=np.float32)
 embedding_matrix[0, dim] = 1
for word, i in tqdm(word_index.items() , disable=not(VERBOSE)) :
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i, :dim] = embedding_vector
continue
return embedding_matrix
def f1(y_true, y_pred):
def recall(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
def f1_smart(y_true, y_pred):
args = np.argsort(y_pred)
 tp = y_true.sum()
fs =(tp - np.cumsum(y_true[args[:-1]])) / np.arange(y_true.shape[0] + tp - 1, tp, -1)
 res_idx = np.argmax(fs)
 return 2 * fs[res_idx],(y_pred[args[res_idx]] + y_pred[args[res_idx + 1]])/ 2<statistical_test>
%%time
embedding_matrix = np.concatenate([build_emb_matrix(word_index, embed_fasttext, FASTTEXT_DIM),
build_emb_matrix(word_index, embed_word2vec, WORD2VEC_DIM)], -1)
 del embed_fasttext, embed_word2vec
_ = gc.collect()
print("Embeddings memory usage", sys.getsizeof(embedding_matrix)/(1024*1024), "MB" )
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale
class Capsule(Layer):
def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(tf.keras.backend.batch_dot(c, u_hat_vecs, [2, 2]))
if i < self.routings - 1:
b = tf.keras.backend.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule )<choose_model_class>
class SimpleReader() :
def __init__(self, file, y_train, batch_size, maxlen, index_filter=lambda x : True):
self.fn = file
self.file = open(file, 'r')
 self.k = 0
self.y_train = y_train
self.batch_size = batch_size
self.maxlen = maxlen
self.index_filter = index_filter
def flow(self):
X = []
y = []
self.k = 0
while True:
line = self.file.readline().replace('
', '')
 if line is not None and len(line)> 0:
if self.index_filter(self.k):
X.append(list(map(int, line.split(' '))))
y.append(self.y_train[self.k])
 self.k += 1
else:
self.file.close()
self.file = open(self.fn, 'r')
 self.k = 0
if len(X)== self.batch_size:
X = pad_sequences(X, maxlen=self.maxlen, truncating='post', padding='post')
 y = np.array(y)
 yield(( X, y))
X = []
y = []
def create_capsule(embedding_matrix):
K.clear_session()
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(rate=0.2 )(x)
 x = Bidirectional(CuDNNGRU(100, return_sequences=True,
kernel_initializer=glorot_normal(seed=12300), recurrent_initializer=orthogonal(gain=1.0, seed=10000)) )(x)
 x = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True )(x)
 x = Flatten()(x)
 x = Dense(100, activation="relu", kernel_initializer=glorot_normal(seed=12300))(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam() ,)
 return model<set_options>
def create_model(embedding_matrix):
input_tensor = Input(shape=(None,))
output_tensor = input_tensor
output_tensor = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1], weights=[embedding_matrix], trainable=False )(output_tensor)
 trainable_embed = Embedding(embedding_matrix.shape[0], TRAINABLE_EMBED_SIZE, trainable=True )(input_tensor)
 output_tensor = Concatenate(-1 )([output_tensor, trainable_embed])
 output_tensor = SpatialDropout1D(0.3 )(output_tensor)
 output_tensor = Bidirectional(CuDNNLSTM(384, return_sequences=True))(output_tensor)
 output_tensor = Add()([Bidirectional(CuDNNGRU(384, return_sequences=True))(output_tensor), output_tensor])
 output_tensor = Add()([Bidirectional(CuDNNGRU(384, return_sequences=True))(output_tensor), output_tensor])
 output_tensor = Concatenate()([GlobalMaxPooling1D()(output_tensor), GlobalAveragePooling1D()(output_tensor)])
 output_tensor = Add()([Dense(384*4, activation='elu' )(output_tensor), output_tensor])
 output_tensor = Dense(CATEGORY_CNT, activation='softmax' )(output_tensor)
 model = Model(inputs=input_tensor, outputs=output_tensor)
 return model
class CyclicLR(Callback):
def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
<set_options>
%%time
train_generator = SimpleReader(".. /input/bonus-x-txt-fasttext-unk/X_train_fasttext_unk.txt", y_train, BATCH_SIZE, MAXLEN, index_filter=lambda x : x % 30 != 7)
 val_generator = SimpleReader(".. /input/bonus-x-txt-fasttext-unk/X_train_fasttext_unk.txt", y_train, BATCH_SIZE, MAXLEN, index_filter=lambda x : x % 30 == 7)
 lr_scheldure = LearningRateScheduler(lambda epoch, lr: EXP_DECAY_COEF ** epoch * 1e-3, verbose=VERBOSE)
 model = create_model(embedding_matrix)
 model.compile(optimizer=Adam(lr=1e-3), loss='sparse_categorical_crossentropy', metrics=['accuracy'])
 model.fit_generator(
 train_generator.flow() ,
validation_data=val_generator.flow() ,
steps_per_epoch=STEPS_PER_EPOCH,
validation_steps=VAL_STEPS_PER_EPOCH,
epochs=EPOCHS,
verbose=VERBOSE,
callbacks=[lr_scheldure]
)
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<define_search_model>
predictions = np.ones(( len(itemid),), dtype=np.int)* -1
gen = SimpleReader(".. /input/bonus-x-txt-fasttext-unk/X_test_fasttext_unk.txt", np.arange(0, len(itemid), 1), PREDICT_BATCH_SIZE, MAXLEN)
 flow = gen.flow()
for i in tqdm(range(( len(itemid)+ PREDICT_BATCH_SIZE - 1)// PREDICT_BATCH_SIZE), disable=not(VERBOSE)) :
batch = next(flow)
 batch_predictions = model.predict_on_batch(batch[0])
 predictions[batch[1]] = batch_predictions.argmax(-1 )
def create_lstm_atten(embedding_matrix):
K.clear_session()
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.1 )(x)
 x = Bidirectional(CuDNNLSTM(40, return_sequences=True))(x)
 y = Bidirectional(CuDNNGRU(40, return_sequences=True))(x)
 atten_1 = Attention(maxlen )(x)
 atten_2 = Attention(maxlen )(y)
 avg_pool = GlobalAveragePooling1D()(y)
 max_pool = GlobalMaxPooling1D()(y)
 conc = concatenate([atten_1, atten_2, avg_pool, max_pool])
 conc = Dense(16, activation="relu" )(conc)
 conc = Dropout(0.1 )(conc)
 outp = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[f1])
 return model<choose_model_class>
submission = pd.read_csv(".. /input/texts-classification-ml-hse-2019/sample_submission.csv")
 submission['Category'] = predictions
submission['Id'] = itemid
def create_cnn(embedding_matrix):
K.clear_session()
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(rate=0.2 )(x)
 conv1 = Conv1D(400, 4, activation = 'relu' )(x)
 conv1 = GlobalMaxPool1D()(conv1)
 out = Dropout(0.5 )(conv1)
 out = Dense(400, activation="relu", kernel_initializer=glorot_normal(seed=12300))(out)
 out = Dense(1, activation="sigmoid" )(out)
 model = Model(inputs=inp, outputs=out)
 model.compile(loss='binary_crossentropy', optimizer=Adam() ,)
 return model<split>
submission['Category'] = submission['Category'].apply(lambda x : b_cat_map[x] )
X_train, X_val, Y_train, Y_val = train_test_split(X, Y, random_state = 2018, test_size = 0.2 )<define_variables>
submission.to_csv("submission.csv", index=False )
val_pred = {}
test_pred = {}<train_model>
submission.to_csv("submission.csv", index=False )