Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<drop_column>
print(os.listdir(".. /input"))
train.drop('X4',axis=1,inplace=True)
 test.drop('X4',axis=1,inplace=True )<data_type_conversions>
EMBEDDING_FILE = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
train = pd.read_csv('.. /input/cleaned-toxic-comments/train_preprocessed.csv')
 test = pd.read_csv('.. /input/cleaned-toxic-comments/test_preprocessed.csv' )
for col in train.columns:
if train[col].dtype == "object":
train[col] = train[col].astype('category')
 train[col] = train[col].cat.codes<data_type_conversions>
train["comment_text"] = train["comment_text"].fillna("fillna")
 test["comment_text"] = test["comment_text"].fillna("fillna")
 X_train = train["comment_text"].str.lower()
y_train = train[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values
X_test = test["comment_text"].str.lower()
for col in test.columns:
if test[col].dtype == "object":
test[col] = test[col].astype('category')
 test[col] = test[col].cat.codes<prepare_x_and_y>
max_features=100000
maxlen=150
embed_size=300
X = train[[col for col in train.columns if col!='y']]
y = train['y']<train_model>
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: {:d} - score: {:.6f}".format(epoch+1, score))
rf = RandomForestRegressor(n_jobs=-1,random_state=42)
 rf.fit(X,y )<define_variables>
tok=text.Tokenizer(num_words=max_features,lower=True)
 tok.fit_on_texts(list(X_train)+list(X_test))
X_train=tok.texts_to_sequences(X_train)
 X_test=tok.texts_to_sequences(X_test)
 x_train=sequence.pad_sequences(X_train,maxlen=maxlen)
 x_test=sequence.pad_sequences(X_test,maxlen=maxlen )
zero_features = list(feature_importances[feature_importances['importance'] == 0.0]['feature'])
 print('There are %d features with 0.0 importance' % len(zero_features))
feature_importances<drop_column>
embeddings_index = {}
with open(EMBEDDING_FILE,encoding='utf8')as f:
for line in f:
values = line.rstrip().rsplit(' ')
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
train.drop(zero_features,axis=1,inplace=True)
 test.drop(zero_features,axis=1,inplace=True )<prepare_x_and_y>
word_index = tok.word_index
num_words = min(max_features, len(word_index)+ 1)
 embedding_matrix = np.zeros(( num_words, embed_size))
for word, i in word_index.items() :
if i >= max_features:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
X = train[[col for col in train.columns if col!='y']]
y = train['y']<train_model>
gru_len = 128
Routings = 5
Num_capsule = 10
Dim_capsule = 16
dropout_p = 0.25
rate_drop_dense = 0.28
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale
class Capsule(Layer):
def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(K.batch_dot(c, u_hat_vecs, [2, 2]))
if i < self.routings - 1:
b = K.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule)
 def get_model() :
input1 = Input(shape=(maxlen,))
embed_layer = Embedding(max_features,
embed_size,
input_length=maxlen,
weights=[embedding_matrix],
trainable=False )(input1)
 embed_layer = SpatialDropout1D(rate_drop_dense )(embed_layer)
 x = Bidirectional(
 GRU(gru_len, activation='relu', dropout=dropout_p, recurrent_dropout=dropout_p, return_sequences=True))(
 embed_layer)
 capsule = Capsule(num_capsule=Num_capsule, dim_capsule=Dim_capsule, routings=Routings,
share_weights=True )(x)
 capsule = Flatten()(capsule)
 capsule = Dropout(dropout_p )(capsule)
 output = Dense(6, activation='sigmoid' )(capsule)
 model = Model(inputs=input1, outputs=output)
 model.compile(
 loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 model.summary()
return model
rf_2 = RandomForestRegressor(n_jobs=-1,random_state=42)
 rf_2.fit(X,y )<train_on_grid>
model = get_model()
batch_size = 256
epochs = 3
X_tra, X_val, y_tra, y_val = train_test_split(x_train, y_train, train_size=0.95, random_state=233)
 RocAuc = RocAucEvaluation(validation_data=(X_val, y_val), interval=1 )
rf_3 = RandomForestRegressor(n_jobs=-1,random_state=42)
 params = {"n_estimators":list(range(50,501,100)) ,
"max_features":[0.5,'sqrt','log2'],
"min_samples_leaf":[1,3,5,10,25],
"min_samples_split":[2,5,7,10,15],
"max_depth":[3,5,7,9]}
r_search = RandomizedSearchCV(estimator=rf_3,param_distributions=params,cv=5,scoring='r2')
 r_search.fit(X,y )<find_best_params>
hist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=1, validation_data=(X_val, y_val),
callbacks=[RocAuc], verbose=1 )
r_search.best_estimator_,r_search.best_params_,r_search.best_score_<define_variables>
hist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=1, validation_data=(X_val, y_val),
callbacks=[RocAuc], verbose=1 )
zero_features = list(feature_importances[feature_importances['importance'] == 0.0]['feature'])
 print('There are %d features with 0.0 importance' % len(zero_features))
feature_importances<drop_column>
hist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=1, validation_data=(X_val, y_val),
callbacks=[RocAuc], verbose=1 )
train.drop(zero_features,axis=1,inplace=True)
 test.drop(zero_features,axis=1,inplace=True )<train_on_grid>
y_pred = model.predict(x_test, batch_size=1024, verbose=1)
<save_to_csv><EOS> 
submission = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv')
 submission[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]] = y_pred
submission.to_csv('submission.csv', index=False)
 model.save_weights('best.hdf5')