Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<load_from_csv>
%matplotlib inline
print(os.listdir(".. /input"))
warnings.filterwarnings('ignore' )
TRAINING = False
USE_FINETUNE = False
FOLDS = 4
SEED = 42
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train = train.query('date > 85' ).reset_index(drop = True)
 train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns})
 train.fillna(train.mean() ,inplace=True)
 train = train.query('weight > 0' ).reset_index(drop = True)
 train['action'] =(( train['resp_1'] > 0)&(train['resp_2'] > 0)&(train['resp_3'] > 0)&(train['resp_4'] > 0)&(train['resp'] > 0)).astype('int')
 features = [c for c in train.columns if 'feature' in c]
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X = train[features].values
y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(train[features[1:]].values,axis=0 )<categorify>
train = pd.read_csv(".. /input/jigsawtraintest/train_jigsaw.csv")
 test= pd.read_csv(".. /input/jigsawtraintest/test_jigsaw.csv")
 EMBEDDING_FILE = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
def create_autoencoder(input_dim,output_dim,noise=0.05):
i = Input(input_dim)
 encoded = BatchNormalization()(i)
 encoded = GaussianNoise(noise )(encoded)
 encoded = Dense(64,activation='relu' )(encoded)
 decoded = Dropout(0.15 )(encoded)
 decoded = BatchNormalization()(decoded)
 decoded = Dense(input_dim,name='decoded' )(decoded)
 x = Dense(32,activation='relu' )(decoded)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 x = Dense(32,activation='relu' )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 x = Dense(output_dim,activation='sigmoid',name='label_output' )(x)
 encoder = Model(inputs=i,outputs=encoded)
 autoencoder = Model(inputs=i,outputs=[decoded,x])
 autoencoder.compile(optimizer=Adam(0.001),loss={'decoded':'mse','label_output':'binary_crossentropy'})
 return autoencoder, encoder<categorify>
train["comment_text"].fillna("fillna")
 test["comment_text"].fillna("fillna")
 X_train = train["comment_text"].str.lower()
y_train = train[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values
X_test = test["comment_text"].str.lower()
def create_model(hp,input_dim,output_dim,encoder):
inputs = Input(input_dim)
 x = encoder(inputs)
 x = Concatenate()([x,inputs, x])
 x = BatchNormalization()(x)
 x = Dropout(hp.Float('init_dropout',0.0,0.5))(x)
 for i in range(hp.Int('num_layers',1,3)) :
x = Dense(hp.Int('num_units_{i}',64,256))(x)
 x = BatchNormalization()(x)
 x = Lambda(tf.keras.activations.swish )(x)
 x = Dropout(hp.Float(f'dropout_{i}',0.0,0.5))(x)
 x = Dense(output_dim,activation='sigmoid' )(x)
 model = Model(inputs=inputs,outputs=x)
 model.compile(optimizer=Adam(hp.Float('lr',0.00001,0.1,default=0.001)) ,loss=BinaryCrossentropy(label_smoothing=hp.Float('label_smoothing',0.0,0.1)) ,metrics=[tf.keras.metrics.AUC(name = 'auc')])
 return model<categorify>
max_features=100000
maxlen=150
embed_size=300
autoencoder, encoder = create_autoencoder(X.shape[-1],y.shape[-1],noise=0.1)
 if TRAINING:
autoencoder.fit(X,(X,y),
epochs=1000,
batch_size=4096,
validation_split=0.1,
callbacks=[EarlyStopping('val_loss',patience=10,restore_best_weights=True)])
 encoder.save_weights('./encoder.hdf5')
 else:
encoder.load_weights('.. /input/fork-of-fork-of-notebookd9779f8f26-f9b834/encoder.hdf5')
 encoder.trainable = False<feature_engineering>
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: {:d} - score: {:.6f}".format(epoch+1, score))
if not TRAINING:
f = np.median
models = models[-2:]
env = janestreet.make_env()
th = 0.5
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<set_options>
tok=text.Tokenizer(num_words=max_features,lower=True)
 tok.fit_on_texts(list(X_train)+list(X_test))
X_train=tok.texts_to_sequences(X_train)
 X_test=tok.texts_to_sequences(X_test)
 x_train=sequence.pad_sequences(X_train,maxlen=maxlen)
 x_test=sequence.pad_sequences(X_test,maxlen=maxlen )
warnings.filterwarnings('ignore')
<feature_engineering>
embeddings_index = {}
with open(EMBEDDING_FILE,encoding='utf8')as f:
for line in f:
values = line.rstrip().rsplit(' ')
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
train = cudf.read_csv('/kaggle/input/jane-street-market-prediction/train.csv')
 features = [c for c in train.columns if 'feature' in c]
f_mean = train[features[1:]].mean()
train = train.query('date > 85' ).reset_index(drop = True)
 train = train.query('weight > 0' ).reset_index(drop = True)
 train[features[1:]] = train[features[1:]].fillna(f_mean)
 train['action'] =(train['resp'] > 0 ).astype('int')
 print("Converting")
 train = train.to_pandas()
f_mean = f_mean.values.get()
np.save('f_mean.npy', f_mean)
<choose_model_class>
word_index = tok.word_index
num_words = min(max_features, len(word_index)+ 1)
 embedding_matrix = np.zeros(( num_words, embed_size))
for word, i in word_index.items() :
if i >= max_features:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
def create_mlp(num_columns, num_labels, hidden_units, dropout_rates, label_smoothing,learning_rate):
inp = tf.keras.layers.Input(shape =(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i+1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation('sigmoid' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 model.compile(optimizer = tfa.optimizers.RectifiedAdam(learning_rate = learning_rate),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics = tf.keras.metrics.AUC(name = 'AUC'), 
)
 return model<define_search_space>
sequence_input = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix],trainable = False )(sequence_input)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(GRU(128, return_sequences=True,dropout=0.1,recurrent_dropout=0.1))(x)
 x = Conv1D(64, kernel_size = 3, padding = "valid", kernel_initializer = "glorot_uniform" )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 x = concatenate([avg_pool, max_pool])
 preds = Dense(6, activation="sigmoid" )(x)
 model = Model(sequence_input, preds)
 model.compile(loss='binary_crossentropy',optimizer=Adam(lr=1e-3),metrics=['accuracy'] )
tf.random.set_seed(2020)
 batch_size = 4096
hidden_units = [160, 160, 160]
dropout_rates = [0.20, 0.20, 0.20, 0.20]
label_smoothing = 1e-2
learning_rate = 1e-3
oof = np.zeros(len(train['action']))
def utility_gain(pred_val,test_index):
uniqueValues = train.loc[test_index,'date'].unique()
min_day = uniqueValues.min().astype('int32')
 max_day = uniqueValues.max().astype('int32')
 nro_days = max_day - min_day +1
print(min_day)
 print(max_day)
 pi = np.zeros(nro_days)
 for j in range(min_day, max_day):
index = train.index[train['date'] == j].intersection(test_index)
 m = train.loc[index, 'weight'] * train.loc[index,'resp'] * pred_val[index]
pi[j-min_day] = np.sum(m)
 t =(np.sum(pi)/ math.sqrt(np.sum(pi**2)))* math.sqrt(250/nro_days)
 utility = min(max(t,0),6)* np.sum(pi)
 return utility
for fold,(tr,te)in enumerate(GroupTimeSeriesSplit().split(train, groups=train['date'])) :
X_tr, X_val = train.loc[tr, features].values, train.loc[te, features].values
y_tr, y_val = train.loc[tr, 'action'].values, train.loc[te, 'action'].values
ckp_path = f'Model_{fold}.hdf5'
model = create_mlp(X_tr.shape[1], 1, hidden_units, dropout_rates, label_smoothing, learning_rate)
 rlr = ReduceLROnPlateau(monitor = 'val_AUC', factor = 0.1, patience = 3, verbose = 0,
min_delta = 1e-4, mode = 'max')
 ckp = ModelCheckpoint(ckp_path, monitor = 'val_AUC', verbose = 0,
save_best_only = True, save_weights_only = True, mode = 'max')
 es = EarlyStopping(monitor = 'val_AUC', min_delta = 1e-4, patience = 5, mode = 'max',
baseline = None, restore_best_weights = True, verbose = 0)
 model.fit(X_tr, y_tr,
validation_data =(X_val, y_val),
epochs = 200,
batch_size = batch_size,
callbacks = [rlr, ckp, es],
verbose = 0)
 oof[te] += model.predict(X_val, batch_size = batch_size * 4 ).ravel()
score = roc_auc_score(y_val, oof[te])
 print(f'Fold {fold} ROC AUC:\t', score)
 pred_val = model.predict(X_val ).ravel()
pred_val = pd.Series(pred_val, index=te)
 pred_val = pred_val.round().astype(int)
 utility = utility_gain(pred_val,te)
 print("%.2f" % utility)
 K.clear_session()
rubbish = gc.collect() 
<compute_test_metric>
from keras.utils.vis_utils import plot_model
score_oof = roc_auc_score(train['action'].values, oof)
 print(score_oof )<load_pretrained>
batch_size = 128
epochs = 5
X_tra, X_val, y_tra, y_val = train_test_split(x_train, y_train, train_size=0.9, random_state=233 )
models = []
for i in range(2):
clf = create_mlp(len(features), 1, hidden_units, dropout_rates,label_smoothing, learning_rate)
 clf.load_weights(f'./Model_{i}.hdf5')
 models.append(clf )<load_pretrained>
model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),callbacks = callbacks_list,verbose=1)
 model.load_weights(filepath)
 print('Predicting.... ')
 y_pred = model.predict(x_test,batch_size=1024,verbose=1 )
f_mean = np.load('./f_mean.npy')
<split>
y_df = np.where(y_pred > 0.5, 1, 0 )
env = janestreet.make_env()
env_iter = env.iter_test() 
<categorify>
y_df = pd.DataFrame(y_df, columns=['toxic','severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate'])
 y_df = y_df.astype('int' )
opt_th = 0.501
for(test_df, pred_df)in tqdm(env_iter):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
for i, clf in enumerate(models):
if i == 0:
pred = clf(x_tt, training = False ).numpy().item() / len(models)
 else:
pred += clf(x_tt, training = False ).numpy().item() / len(models)
 pred_df.action = np.where(pred >= opt_th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<load_from_csv>
labels = ['toxic','severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
df=pd.read_csv('.. /input/train.csv')
 %matplotlib inline<prepare_x_and_y>
def get_metri_scores(y_test, y_test_pred):
vals = precision_recall_fscore_support(y_test, y_test_pred, average='macro')
 precision = vals[0]
recall = vals[1]
f1 = vals[2]
acc = accuracy_score(y_test, y_test_pred)
 return precision, recall, f1, acc
y_train = df.pop("type")
 df = pd.get_dummies(df)
 print(df.columns )<prepare_x_and_y>
results_cv = pd.DataFrame({'labels': labels})
 results_cv['acc'] = 0
results_cv['f1'] = 0
results_cv['precision'] = 0
results_cv['recall'] = 0
for col in labels:
print(col)
 precision, recall, f1, acc = get_metri_scores(test[col], y_df[col])
 results_cv['acc'][results_cv['labels']==col] = acc
results_cv['f1'][results_cv['labels']==col] = f1
results_cv['precision'][results_cv['labels']==col] = precision
results_cv['recall'][results_cv['labels']==col] = recall
features = ['bone_length', 'rotting_flesh', 'hair_length', 'has_soul',
'color_black', 'color_blood', 'color_blue', 'color_clear',
'color_green', 'color_white']
X = df[features]
id_ = df["id"] 
<prepare_x_and_y>
sequence_input = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix],trainable = False )(sequence_input)
 x = SpatialDropout1D(0.2 )(x)
 x = LSTM(256, return_sequences=True,dropout=0.1,recurrent_dropout=0.1 )(x)
 x = Conv1D(64, kernel_size = 3, padding = "valid", kernel_initializer = "glorot_uniform" )(x)
 max_pool = GlobalMaxPooling1D()(x)
 x = Dense(128, activation='relu' )(max_pool)
 x = Dropout(0.1 )(x)
 preds = Dense(6, activation="sigmoid" )(x)
 model2 = Model(sequence_input, preds)
 model2.compile(loss='binary_crossentropy',optimizer=Adam(lr=1e-3),metrics=['accuracy'] )
features = ["bone_length","rotting_flesh","hair_length","color","has_soul"]
X = df[features]
y = df["type"]<categorify>
filepath="weights_novice_model.hdf5"
checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
 early = EarlyStopping(monitor="val_acc", mode="max", patience=5)
 ra_val = RocAucEvaluation(validation_data=(X_val, y_val), interval = 1)
 callbacks_list = [ra_val,checkpoint, early]
model2.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),callbacks = callbacks_list,verbose=1)
 model2.load_weights(filepath)
 print('Predicting.... ')
 y_pred = model2.predict(x_test,batch_size=1024,verbose=1 )
letype = LE()
y = letype.fit_transform(y_train)
 lecolor = LE() 
<set_options>
y_df = np.where(y_pred > 0.5, 1, 0)
 y_df = pd.DataFrame(y_df, columns=['toxic','severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate'])
 y_df = y_df.astype('int' )
<train_model><EOS> 
def get_metri_scores(y_test, y_test_pred):
vals = precision_recall_fscore_support(y_test, y_test_pred, average='macro')
 precision = vals[0]
recall = vals[1]
f1 = vals[2]
acc = accuracy_score(y_test, y_test_pred)
 return precision, recall, f1, acc
results_cv = pd.DataFrame({'labels': labels})
 results_cv['acc'] = 0
results_cv['f1'] = 0
results_cv['precision'] = 0
results_cv['recall'] = 0
for col in labels:
print(col)
 precision, recall, f1, acc = get_metri_scores(test[col], y_df[col])
 results_cv['acc'][results_cv['labels']==col] = acc
results_cv['f1'][results_cv['labels']==col] = f1
results_cv['precision'][results_cv['labels']==col] = precision
results_cv['recall'][results_cv['labels']==col] = recall