Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<compute_test_metric>
np.random.seed(42)
 warnings.filterwarnings('ignore')
 os.environ['OMP_NUM_THREADS'] = '4'
eng_stopwords = set(stopwords.words("english"))
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))<choose_model_class>
from keras.models import Sequential
from keras.layers import Dense, Activation
def create_model(num_columns):
model = tf.keras.Sequential([
tf.keras.layers.Input(num_columns),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation="relu")) ,
tf.keras.layers.Dropout(0.484631),
tf.keras.layers.BatchNormalization() ,
tfa.layers.WeightNormalization(tf.keras.layers.Dense(4096, activation="relu")) ,
tf.keras.layers.Dropout(0.484631),
tf.keras.layers.BatchNormalization() ,
tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation="relu")) ,
tf.keras.layers.Dropout(0.484631),
tf.keras.layers.BatchNormalization() ,
tfa.layers.WeightNormalization(tf.keras.layers.Dense(4096, activation="relu")) ,
tf.keras.layers.Dropout(0.484631),
tf.keras.layers.BatchNormalization() ,
tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation="relu")) ,
tf.keras.layers.Dropout(0.484631),
tf.keras.layers.BatchNormalization() ,
tfa.layers.WeightNormalization(tf.keras.layers.Dense(4096, activation="relu")) ,
tf.keras.layers.Dropout(0.484631),
tf.keras.layers.BatchNormalization() ,
tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation="sigmoid"))
])
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam() , sync_period=10),
loss=losses.BinaryCrossentropy(label_smoothing=0.001),metrics=logloss 
)
 return model<define_variables>
print("reading train files")
 train = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv',encoding='utf-8')
 train = train.replace(r'
',' ', regex=True)
 train = train.replace(r'\',' ', regex=True)
 print(train.head())
 print("Now reading test files")
 test=pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv',encoding='utf-8')
 ’ ).
 ]',re.IGNORECASE)
 special_character_removal = re.compile(r'[^A-Za-z\.\-\?\!\,\
def clean_text(x):
x_ascii = unidecode(x)
 x_clean = special_character_removal.sub('',x_ascii)
 return x_clean
train['clean_text'] = train['comment_text'].apply(lambda x: clean_text(str(x)))
 test['clean_text'] = test['comment_text'].apply(lambda x: clean_text(str(x)))
 X_train = train['clean_text'].fillna("something" ).values
y_train = train[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values
X_test = test['clean_text'].fillna("something" ).values
top_feats = range(1039)
 print(len(top_feats))<normalization>
def add_features(df):
df['comment_text'] = df['comment_text'].apply(lambda x:str(x))
df['total_length'] = df['comment_text'].apply(len)
 df['capitals'] = df['comment_text'].apply(lambda comment: sum(1 for c in comment if c.isupper()))
df['caps_vs_length'] = df.apply(lambda row: float(row['capitals'])/float(row['total_length']),
axis=1)
 df['num_words'] = df.comment_text.str.count('\S+')
 df['num_unique_words'] = df['comment_text'].apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique'] = df['num_unique_words'] / df['num_words']
return df
train = add_features(train)
 test = add_features(test)
 features = train[['caps_vs_length', 'words_vs_unique']].fillna(0)
 test_features = test[['caps_vs_length', 'words_vs_unique']].fillna(0)
 ss = StandardScaler()
ss.fit(np.vstack(( features, test_features)))
 features = ss.transform(features)
 test_features = ss.transform(test_features )
scaler = StandardScaler()
X = scaler.fit_transform(train.values)
 x_tt = scaler.transform(test_features.values )<data_type_conversions>
train_comb = train.groupby(COLUMNS)\ 
.size() \ 
.sort_values(ascending=False)\ 
.reset_index() \ 
.rename(columns={0: 'count'})
 train_comb.head(n=10 )
train = train.astype('float')
 test = test.astype('float' )<define_variables>
train[COLUMNS].corr().abs().unstack().sort_values(ascending=False )
p_min = 0.001
p_max = 0.999<init_hyperparams>
word_counter = {}
def clean_text(text):
text = re.sub('[{}]'.format(string.punctuation), ' ', text.lower())
 return ' '.join([word for word in text.split() if word not in(eng_stopwords)])
 for categ in CATEGORIES:
d = Counter()
train[train[categ] == 1]['comment_text'].apply(lambda t: d.update(clean_text(t ).split()))
word_counter[categ] = pd.DataFrame.from_dict(d, orient='index')\ 
.rename(columns={0: 'count'})\ 
.sort_values('count', ascending=False)
 for w in word_counter:
wc = word_counter[w]
wordcloud = WordCloud(
 background_color='black',
max_words=200,
max_font_size=100,
random_state=4561
).generate_from_frequencies(wc.to_dict() ['count'])
 fig = plt.figure(figsize=(12, 8))
plt.title(w)
 plt.imshow(wordcloud)
 plt.axis('off')
 plt.show()
N_STARTS = 1
tf.random.set_seed(42)
 ss_NN = ss_svc.copy()
res_NN = train_targets.copy()
ss_NN.loc[:, train_targets.columns] = 0
res_NN.loc[:, train_targets.columns] = 0
for seed in range(N_STARTS):
for n,(tr, te)in enumerate(MultilabelStratifiedKFold(n_splits=12, random_state=seed, shuffle=True ).split(train_targets, train_targets)) :
print(f'Fold {n}')
 model = create_model(len(top_feats))
checkpoint_path = f'repeat:{seed}_Fold:{n}.hdf5'
reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, verbose=1, epsilon=1e-4, mode='min')
 cb_checkpt = ModelCheckpoint(checkpoint_path, monitor = 'val_loss', verbose = 0, save_best_only = True,
save_weights_only = True, mode = 'min')
 model.fit(train.values[tr][:, top_feats],
train_targets.values[tr],
validation_data=(train.values[te][:, top_feats], train_targets.values[te]),
epochs=35, batch_size=128,
callbacks=[reduce_lr_loss, cb_checkpt], verbose=2, 
)
 model.load_weights(checkpoint_path)
 test_predict = model.predict(test.values[:, top_feats])
 val_predict = model.predict(train.values[te][:, top_feats])
 ss_NN.loc[:, train_targets.columns] += test_predict
res_NN.loc[te, train_targets.columns] += val_predict
print('')
 ss_NN.loc[:, train_targets.columns] /=(( n+1)* N_STARTS)
 res_NN.loc[:, train_targets.columns] /= N_STARTS<feature_engineering>
max_features=20000
maxlen = 50
tokenizer = text.Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(X_train)+ list(X_test))
X_train_sequence = tokenizer.texts_to_sequences(X_train)
 X_test_sequence = tokenizer.texts_to_sequences(X_test)
 x_train = sequence.pad_sequences(X_train_sequence, maxlen=maxlen)
 x_test = sequence.pad_sequences(X_test_sequence, maxlen=maxlen)
ss_NN.loc[:, train_targets.columns] = np.clip(ss_NN[train_targets.columns].values,p_min,p_max)
 res_NN.loc[:, train_targets.columns] = np.clip(res_NN[train_targets.columns].values,p_min,p_max)
 print(f'NN OOF Metric: {log_loss_metric(train_targets, res_NN)}')
 res_NN.loc[train['cp_type'] == 1, train_targets.columns] = 0
ss_NN.loc[test['cp_type'] == 1, train_targets.columns] = 0
print(f'NN OOF Metric with postprocessing: {log_loss_metric(train_targets, res_NN)}' )<save_to_csv>
max_features=20000
maxlen = 50
ss_NN.to_csv('submission.csv', index = False )<set_options>
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_vectors = np.zeros(( nb_words,501))
def load_embedding(filename):
file = open(filename,'r')
 lines = file.readlines()
file.close()
embedding = dict()
for line in lines:
parts = line.split()
embedding[parts[0]] = np.asarray(parts[1:], dtype='float32')
 return embedding
def get_weight_matrix(embedding, vocab):
vocab_size = len(vocab)+ 1
weight_matrix = np.zeros(( vocab_size, 200))
for word, i in vocab.items() :
vector = embedding.get(word)
 if vector is not None:
weight_matrix[i] = vector
return weight_matrix
raw_embedding = load_embedding('.. /input/glove-twitter-27b-200d-txt/glove.twitter.27B.200d.txt')
 embedding_vectors = get_weight_matrix(raw_embedding, tokenizer.word_index)[:nb_words,:]
warnings.filterwarnings('ignore')
<load_from_csv>
raw_embedding['hate']
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<load_from_csv>
print("Base Accuracy - Predicting all labels as non toxic ")
(1-train_comb[['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']].mean())*100.0
scored = train_targets_scored.copy()
drug = pd.read_csv('.. /input/lish-moa/train_drug.csv')
 targets = scored.columns[1:]
scored = scored.merge(drug, on='sig_id', how='left')
 vc = scored.drug_id.value_counts()
vc1 = vc.loc[vc<=18].index.sort_values()
vc2 = vc.loc[vc>18].index.sort_values()
SEED = 42
FOLDS = 7
dct1 = {}; dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=FOLDS, shuffle=True,
random_state=SEED)
 tmp = scored.groupby('drug_id')[targets].mean().loc[vc1]
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=FOLDS, shuffle=True,
random_state=SEED)
 tmp = scored.loc[scored.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 scored['kfold'] = scored.drug_id.map(dct1)
 scored.loc[scored.kfold.isna() ,'kfold'] = scored.loc[scored.kfold.isna() ,'sig_id'].map(dct2)
 scored.kfold = scored.kfold.astype('int8')
 train_targets_scored = scored.copy()
del scored<define_variables>
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
self.max_score = 0
self.not_better_count = 0
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=1)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: %d - score: %.6f
" %(epoch+1, score))
if(score > self.max_score):
print("*** New High Score(previous: %.6f)
 " % self.max_score)
 model.save_weights("best_weights.h5")
 self.max_score=score
self.not_better_count = 0
else:
self.not_better_count += 1
if self.not_better_count > 3:
print("Epoch %05d: early stopping, high score = %.6f" %(epoch,self.max_score))
self.model.stop_training = True
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<normalization>
def get_model(features,clipvalue=1.,num_filters=40,dropout=0.5,embed_size=200):
features_input = Input(shape=(features.shape[1],))
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_vectors], trainable=False,name='EmbeddingLayer' )(inp)
 x, x_h, x_c = Bidirectional(GRU(num_filters, return_sequences=True, return_state = True),name='BidirectionalGRU' )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 x = concatenate([avg_pool, x_h, max_pool,features_input])
 outp = Dense(6, activation="sigmoid" )(x)
 model = Model(inputs=[inp,features_input], outputs=outp)
 adam = optimizers.adam(clipvalue=clipvalue)
 model.compile(loss='binary_crossentropy',
optimizer=adam,
metrics=['accuracy'])
 return model
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=125,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<set_options>
model = get_model(features)
 batch_size = 32
epochs = 5
gc.collect()
K.clear_session()
num_folds = 5
predict = np.zeros(( test.shape[0],6))
scores = []
oof_predict = np.zeros(( train.shape[0],6))
kf = KFold(n_splits=num_folds, shuffle=True, random_state=239)
 for train_index, test_index in kf.split(x_train):
kfold_y_train,kfold_y_test = y_train[train_index], y_train[test_index]
kfold_X_train = x_train[train_index]
kfold_X_features = features[train_index]
kfold_X_valid = x_train[test_index]
kfold_X_valid_features = features[test_index]
gc.collect()
K.clear_session()
model = get_model(features)
 ra_val = RocAucEvaluation(validation_data=([kfold_X_valid,kfold_X_valid_features], kfold_y_test), interval = 1)
 model.fit([kfold_X_train,kfold_X_features], kfold_y_train, batch_size=batch_size, epochs=epochs, verbose=1,
callbacks = [ra_val])
 gc.collect()
model.load_weights("best_weights.h5")
 predict += model.predict([x_test,test_features], batch_size=batch_size,verbose=1)/ num_folds
gc.collect()
oof_predict[test_index] = model.predict([kfold_X_valid, kfold_X_valid_features],batch_size=batch_size, verbose=1)
 cv_score = roc_auc_score(kfold_y_test, oof_predict[test_index])
 scores.append(cv_score)
 print('Cross Validation Score: ',cv_score)
 print("Model Completion for Keras DL")
 print('Total CV score is {}'.format(np.mean(scores)))
 print("Saving Predictions for offline upload")
 sample_submission = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv")
 class_names = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
sample_submission[class_names] = predict
sample_submission.to_csv('UdactiyAssignment_v1.csv',index=False)
<concatenate><EOS> 
print("Code Run Completed" )