Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<compute_train_metric>
import numpy as np
import pandas as pd
from keras.layers import Dense,Input,Bidirectional,Conv1D,GRU
from keras.layers import Embedding,GlobalMaxPooling1D, GlobalAveragePooling1D, concatenate, SpatialDropout1D
from keras.preprocessing import text, sequence
from keras import initializers, regularizers, constraints, optimizers, layers, callbacks
from keras.callbacks import EarlyStopping,ModelCheckpoint
from keras.models import Model
from keras.optimizers import Adam, SGD
from sklearn.model_selection import train_test_split
import spacy
pca = PCA(n_components=512)
 pca.fit(genes_train)
 pca_error = mean_squared_error(genes_test,pca.inverse_transform(pca.transform(genes_test)))
 print('PCA Reconstruction Error for Genes is ' + str(pca_error))<concatenate>
glove_file = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
train_file = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv.zip')
 test_file = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv.zip')
 X_train = train_file["comment_text"].str.lower()
X_test = test_file["comment_text"].str.lower()
y_train = train_file[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values
data_train = np.concatenate(( data_train[:,~(cs+gs)],ae_genes_train,ae_cells_train),axis=1)
 data_test = np.concatenate(( data_test[:,~(cs+gs)],ae_genes_test,ae_cells_test),axis=1 )<compute_test_metric>
max_features=150000
maxlen=200
embed_size=300
tok=text.Tokenizer(num_words=max_features,lower=True)
 tok.fit_on_texts(list(X_train)+list(X_test))
X_train=tok.texts_to_sequences(X_train)
 X_test=tok.texts_to_sequences(X_test)
 x_train=sequence.pad_sequences(X_train,maxlen=maxlen)
 x_test=sequence.pad_sequences(X_test,maxlen=maxlen )
n_labels = train_targets_scored.shape[1]
n_features = data_train.shape[1]
n_train = data_train.shape[0]
n_test = data_test.shape[0]
p_min = 0.0005
p_max = 0.9995
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))
n_seeds = 3
seeds = [34,9,18]
n_folds = 5
y_pred = np.zeros(( n_test,n_labels))
oof = tf.constant(0.0)
 hists = []
bias = tf.keras.initializers.Constant(np.log(labels_train.mean(axis=0)))
 for seed in seeds:
fold = 0
mskf = MultilabelStratifiedKFold(n_splits=n_folds,shuffle=True,random_state=seed)
 for train, test in mskf.split(data_train,labels_train):
X_train = data_train[train]
X_test = data_train[test]
y_train = labels_train[train]
y_test = labels_train[test]
model = Sequential()
model.add(layers.Dropout(0.3))
model.add(layers.Dense(1536))
model.add(layers.Activation('elu'))
model.add(layers.BatchNormalization())
 model.add(layers.Dropout(0.7))
model.add(layers.Dense(1024))
model.add(layers.Activation('elu'))
model.add(layers.BatchNormalization())
 model.add(layers.Dropout(0.5))
model.add(layers.Dense(512))
model.add(layers.Activation('elu'))
model.add(layers.BatchNormalization())
 model.add(layers.Dropout(0.3))
model.add(layers.Dense(n_labels,activation='sigmoid',bias_initializer=bias))
model.compile(optimizer=optimizers.Adam(learning_rate=1E-5), loss=losses.BinaryCrossentropy(label_smoothing=0.001), metrics=['binary_crossentropy',logloss])
 reduce_lr = callbacks.ReduceLROnPlateau(monitor='val_logloss', factor=0.3, patience=5, mode='min', min_lr=1E-5)
 early_stopping = callbacks.EarlyStopping(monitor='val_logloss', min_delta=1E-5, patience=24, mode='min',restore_best_weights=True)
 def scheduler(epoch,lr):
if epoch%16<9:
lr += np.exp(-int(epoch/16)) *(0.5*1E-2)/16
else:
lr -= np.exp(-int(epoch/16)) *(0.5*1E-2)/16
return lr
lr_scheduler = callbacks.LearningRateScheduler(scheduler)
 hist = model.fit(X_train,y_train, batch_size=128, epochs=192,verbose=0,validation_data =(X_test,y_test),callbacks=[lr_scheduler, early_stopping])
 model.compile(optimizer=optimizers.Adam(learning_rate=1E-5), loss=losses.BinaryCrossentropy(label_smoothing=0.001), metrics=['binary_crossentropy',logloss])
 model.fit(X_train,y_train, batch_size=128, epochs=192,verbose=0,validation_data =(X_test,y_test),callbacks=[early_stopping])
 hists.append(hist)
 model.save('AutoEncoded_seed_'+str(seed)+'_fold_'+str(fold))
y_val = model.predict(X_test)
 oof += logloss(tf.constant(y_test,dtype=tf.float32),tf.constant(y_val,dtype=tf.float32)) /(n_folds*n_seeds)
 y_pred += model.predict(data_test)/(n_folds*n_seeds)
 fold += 1<save_to_csv>
embeddings_index = {}
counter = 0
with open(glove_file,encoding='utf8')as f:
for line in f:
counter += 1
if(counter%1000000)==0:
print(counter)
 values = line.rstrip().rsplit(' ')
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
p_min = 0.0005
p_max = 0.9995
sub = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub.iloc[:,1:] = np.clip(y_pred,p_min,p_max)
 sub.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub.to_csv('submission.csv', index=False )<import_modules>
nlp = spacy.load('en');
word_index = tok.word_index
num_words = min(max_features, len(word_index)+ 1)
 embedding_matrix = np.random.randn(num_words, embed_size)/4
kk = 0
moo = 0
for word, i in word_index.items() :
if i >= max_features:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
kk += 1
else:
for x in nlp(word,disable=['parser', 'ner']):
embedding_vector = embeddings_index.get(x.lemma_)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
kk += 1
break
import numpy as np
import pandas as pd
import os 
<load_from_csv>
model_input = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix],trainable = False )(model_input)
 x = SpatialDropout1D(0.1 )(x)
 x = Bidirectional(GRU(200, return_sequences=True,dropout=0.25,recurrent_dropout=0.25,implementation=1))(x)
 x = Conv1D(128, kernel_size = 3 )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 x = concatenate([avg_pool, max_pool])
 preds = Dense(6, activation="sigmoid" )(x)
 model = Model(model_input, preds)
 model.summary()
model.compile(loss='binary_crossentropy',optimizer=Adam(lr=2e-4),metrics=['accuracy'])
submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 public_ids = list(submission['sig_id'].values)
 submission.shape<load_from_csv>
model.fit(x_train, y_train, batch_size=128, epochs=10, verbose=1 )
<load_from_csv><EOS> 
y_pred = model.predict(x_test,batch_size=1024,verbose=1)
 submission = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv.zip')
 submission[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]] = y_pred
submission.to_csv('submission.csv', index=False )