Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<compute_train_metric>
print(tf.__version__)
 tf.test.is_gpu_available(
 cuda_only=False,
min_cuda_compute_capability=None
)
def learningRateAnnl(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.995, current_iter)
 return max(lr, min_learning_rate)
 def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) ,-1 ).argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', f1, True )<train_model>
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt'
]
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220
TEXT_COLUMN = 'comment_text'
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'
fit_params={"early_stopping_rounds":300,
"eval_metric" : evaluate_macroF1_lgb,
"eval_set" : [(xvalid3, yvalid3.copy() -1)],
'eval_names': ['valid'],
'callbacks': [lgb.reset_parameter(learning_rate=learningRateAnnl)],
'verbose': False,
'categorical_feature': 'auto'}<define_search_space>
train_df = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 test_df = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv')
 submission = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv")
 y = train_df[list_classes].values
x_train = train_df[TEXT_COLUMN].astype(str)
 y_train = y
x_test = test_df[TEXT_COLUMN].astype(str )
param_test ={'num_leaves': randint(12, 20),
'min_child_samples': randint(40, 120),
'subsample': uniform(loc=0.75, scale=0.20),
'colsample_bytree': uniform(loc=0.8, scale=0.15),
}<choose_model_class>
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f)
 def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix
def build_model(embedding_matrix):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(6, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=result)
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model
maxHPs = 400
classifier = lgb.LGBMClassifier(learning_rate=0.05, n_jobs=-1, n_estimators=500, objective='multiclass')
<init_hyperparams>
%%time
tokenizer = text.Tokenizer(filters=CHARS_TO_REMOVE)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )
opt_parameters = {'colsample_bytree': 0.8755593602517565,
'min_child_samples': 51,
'num_leaves': 19,
'subsample': 0.9437154452377117}<train_model>
%%time
embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1 )
classifier = lgb.LGBMClassifier(**classifier.get_params())
 classifier.set_params(**opt_parameters)
 fit_params['verbose'] = 200
_ = classifier.fit(xtrain3,(ytrain3 ).copy() -1, **fit_params )<train_model>
X_train, X_valid, Y_train, Y_valid = train_test_split(x_train, y_train, test_size = 0.1)
kfold = 5
kf = StratifiedKFold(n_splits=kfold, shuffle=True)
 for trn_idx, tst_idx in kf.split(train.drop(['Target'], axis=1), train['Target']):
xtr, xval = train.drop(['Target'], axis=1 ).iloc[trn_idx], train.drop(['Target'], axis=1 ).iloc[tst_idx]
ytr, yval = train['Target'].iloc[trn_idx].copy() -1, train['Target'].iloc[tst_idx].copy() -1
classifier.fit(xtr, ytr, eval_set=[(xval, yval)],
early_stopping_rounds=300, verbose=200 )<predict_on_test>
EPOCHS = 5
SEEDS = 10
pred = 0
for ii in range(SEEDS):
model = build_model(embedding_matrix)
 for global_epoch in range(EPOCHS):
print(global_epoch)
 model.fit(
 X_train,
Y_train,
validation_data =(X_valid, Y_valid),
batch_size=128,
epochs=1,
verbose=2,
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.55 ** global_epoch))
] 
)
 val_preds_3 = model.predict(X_valid)
 AUC = 0
for i in range(6):
AUC += roc_auc_score(Y_valid[:,i], val_preds_3[:,i])/6.
 print(AUC)
 pred += model.predict(x_test, batch_size = 1024, verbose = 1)/SEEDS
model.save_weights('model_weights_'+str(ii)+'.h5')
 os.system('gzip '+'model_weights_'+str(ii)+'.h5')
<train_model><EOS> 
submission[list_classes] =(pred)
 submission.to_csv("submission.csv", index = False )