Kaggle competition: Toxic Comment Classification Challenge
app_train['is_train'] = 1
app_train['is_test'] = 0
app_test['is_train'] = 0
app_test['is_test'] = 1
print("
Joining the training(app_train)and testing(app_test)dataset for pre-processing into pandas DataFrame 'data'.")
 data = pd.concat([app_train, app_test], axis=0)
 print("app_train has {0:,} samples and {1} features.".format(app_train.shape[0], app_train.shape[1]-4))
print("app_test has {0:,} samples and {1} features.".format(app_test.shape[0], app_test.shape[1]-3))
print("data has {0:,} samples and {1} features BEFORE one-hot encoding.".format(data.shape[0], data.shape[1]-4))
assert(data.shape[0] == app_train.shape[0] + app_test.shape[0])
 assert(data.shape[1] >= max(app_train.shape[1], app_test.shape[1]))<categorify>
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
def _one_hot_encoding(data):
return pd.get_dummies(data)
 print("
Performing one-hot encoding on {} dataset.".format('data'))
data = _one_hot_encoding(data)
 print("app has {0:,} samples and {1} features AFTER one-hot encoding.".format(data.shape[0], data.shape[1]-4))
posc_bal = _one_hot_encoding(posc_bal)
 prev_app = _one_hot_encoding(prev_app)
 inst_pay = _one_hot_encoding(inst_pay)
 cc_bal = _one_hot_encoding(cc_bal)
 bureau = _one_hot_encoding(bureau )<create_dataframe>
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.models import Model
from keras.layers import Input, Dense, Embedding, SpatialDropout1D, concatenate, GRU, Bidirectional, GlobalAveragePooling1D, GlobalMaxPooling1D
from keras.callbacks import Callback, ModelCheckpoint, EarlyStopping
import keras.backend as K
data_train_test = data.copy()<feature_engineering>
EMBEDDING_FILE = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
print("Merge 'Point of Sale Cash Balance' dataset.")
 posc_bal_count = posc_bal[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
posc_bal['POSC_BAL_COUNT'] = posc_bal['SK_ID_CURR'].map(posc_bal_count['SK_ID_PREV'])
 posc_bal = posc_bal.drop(['SK_ID_PREV'], axis=1)
 posc_bal_avg = posc_bal.groupby('SK_ID_CURR' ).mean()
posc_bal_avg.columns = ['pcb_' + col for col in posc_bal_avg.columns]
data = data.merge(right=posc_bal_avg.reset_index() , how='left', on='SK_ID_CURR' )<feature_engineering>
train = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 test = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv')
 submission = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv' )
<merge>
X_train = train['comment_text'].values
y_train = train[['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']].values
X_test = test['comment_text'].values
print("Merge 'Previous Applications' dataset.")
 prev_app_count = prev_app[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
prev_app['PREV_COUNT'] = prev_app['SK_ID_CURR'].map(prev_app_count['SK_ID_PREV'])
 prev_app = prev_app.drop(['SK_ID_PREV'], axis=1)
 prev_app_avg = prev_app.groupby('SK_ID_CURR' ).mean()
prev_app_avg.columns = ['pa_' + col for col in prev_app_avg.columns]
data = data.merge(right=prev_app_avg.reset_index() , how='left', on='SK_ID_CURR' )<merge>
unique_words = {word for sent in X_train for word in sent.split() }.union({word for sent in X_test for word in sent.split() })
 print(f'number of unique words in the corpus {len(unique_words)}' )
print("Merge 'Installments Payments' dataset.")
 inst_pay_count = inst_pay[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
inst_pay['INST_PAY_COUNT'] = inst_pay['SK_ID_CURR'].map(inst_pay_count['SK_ID_PREV'])
 inst_pay = inst_pay.drop(['SK_ID_PREV'], axis=1)
 inst_pay_avg = inst_pay.groupby('SK_ID_CURR' ).mean()
inst_pay_avg.columns = ['ip_' + col for col in inst_pay_avg.columns]
data = data.merge(right=inst_pay_avg.reset_index() , how='left', on='SK_ID_CURR' )<merge>
max(map(lambda x: len(x.split()), X_train))
print("Merge 'Credit Card Balance' dataset.")
 cc_bal_count = cc_bal[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
cc_bal['CC_BAL_COUNT'] = cc_bal['SK_ID_CURR'].map(cc_bal_count['SK_ID_PREV'])
 cc_bal = cc_bal.drop(['SK_ID_PREV'], axis=1)
 cc_bal_avg = cc_bal.groupby('SK_ID_CURR' ).mean()
cc_bal_avg.columns = ['ccb_' + col for col in cc_bal_avg.columns]
data = data.merge(right=cc_bal_avg.reset_index() , how='left', on='SK_ID_CURR' )<merge>
max_words = 40000
max_len = 100
embedding_size = 300
print("Merge 'Bureau' dataset.")
 bureau_count = bureau[['SK_ID_CURR', 'SK_ID_BUREAU']].groupby('SK_ID_CURR' ).count()
bureau['BUREAU_COUNT'] = bureau['SK_ID_CURR'].map(bureau_count['SK_ID_BUREAU'])
 bureau = bureau.drop(['SK_ID_BUREAU'], axis=1)
 bureau_avg = bureau.groupby('SK_ID_CURR' ).mean()
bureau_avg.columns = ['b_' + col for col in bureau_avg.columns]
data = data.merge(right=bureau_avg.reset_index() , how='left', on='SK_ID_CURR' )<feature_engineering>
tokenizer = Tokenizer(num_words=max_words)
 tokenizer.fit_on_texts(list(X_train)+ list(X_test))
for token, idx in zip(list(tokenizer.word_index.keys())[:5], list(tokenizer.word_index.values())[:5]):
print(f'tokenized {token} ------> {idx}' )
<import_modules>
X_train_tokenized = tokenizer.texts_to_sequences(X_train)
 X_test_tokenized = tokenizer.texts_to_sequences(X_test)
 for text, tokenized in zip(X_train[:5], X_train_tokenized[:5]):
print(f'{text.split() [:5]}...------> {tokenized[:5]}...')
 X_train = X_train_tokenized
X_test = X_test_tokenized
scaler = MinMaxScaler()
numerical = ['DAYS_BIRTH', 'AMT_CREDIT', 'DAYS_ID_PUBLISH', 'DAYS_REGISTRATION', 'DAYS_EMPLOYED']
data[numerical] = scaler.fit_transform(data[numerical] )<create_dataframe>
X_train_paddded = pad_sequences(X_train, maxlen=max_len, padding='post')
 X_test_paddded = pad_sequences(X_test, maxlen=max_len, padding='post')
 for train_unpadded, train_padded in zip(X_train[:2], X_train_paddded[:2]):
print(f'{train_unpadded} ------> {len(train_unpadded)} values + {max_len - len(train_unpadded)} zeros')
 X_train = X_train_paddded
X_test = X_test_paddded
data_to_use = 'ALL'
if data_to_use == 'data_train_test':
data = data_train_test.copy()<count_missing_values>
word_index = tokenizer.word_index
tokenized_words = set(list(word_index.keys())[:max_words])
 embedding_matrix = np.zeros(( len(tokenized_words)+1, embedding_size))
found = 0
with open(EMBEDDING_FILE)as f:
for line in f:
word, coord = line.split(' ', 1)
 if word in tokenized_words:
embedding_matrix[word_index[word]] = np.asarray(coord.split() , dtype='float32')
 found += 1
print(f'found {found} of {len(tokenized_words)} words' )
print("
Filling NaN values in the dataset using pandas.fillna() using the column mean() value.")
 print("Number of NaN values in the dataset BEFORE running pandas.fillna() : {:,}".format(data.isnull().sum().sum()))
data = data.fillna(data.mean())
 nan_after = data.isnull().sum().sum()
print("Number of NaN values in the dataset AFTER running pandas.fillna() : {:,}".format(nan_after))
assert(nan_after == 0 )<set_options>
class RocAucCallback(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score - roc_auc_score(self.y_val, y_pred)
 print(f'
ROC-AUC - epoch {epoch+1} - score: {score}' )
del posc_bal, posc_bal_count, posc_bal_avg, bureau_bal, app_train, app_test
del prev_app, prev_app_count, prev_app_avg, inst_pay, inst_pay_count, inst_pay_avg, cc_bal, cc_bal_count, cc_bal_avg
del bureau, bureau_count, bureau_avg, data_train_test
gc.collect()<drop_column>
class GlobalAveragePooling1DMasked(GlobalAveragePooling1D):
def call(self, x, mask=None):
if mask != None:
return K.sum(x, axis=1)/ K.clip(K.tf.cast(K.tf.count_nonzero(x, axis=1), dtype=K.tf.float32), 1, K.int_shape(x)[1])
 else:
return super().call(x )
print("
Separating the training and testing dataset after completing pre-processing.")
 train = data[data['is_train'] == 1]
target = train['TARGET']
train = train.drop(['TARGET', 'SK_ID_CURR', 'is_test', 'is_train'], axis=1)
 test = data[data['is_test'] == 1]
test_id = test['SK_ID_CURR']
test = test.drop(['TARGET', 'SK_ID_CURR', 'is_test', 'is_train'], axis=1)
 print("train has {:,} samples and {} features.".format(train.shape[0], train.shape[1]))
print("test has {:,} samples and {} features.".format(test.shape[0], test.shape[1]))<split>
def get_model() :
inp = Input(shape=(max_len,))
x = Embedding(max_words+1, embedding_size, weights=[embedding_matrix], mask_zero=True )(inp)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(GRU(80, return_sequences=True))(x)
 avg_pool = GlobalAveragePooling1DMasked()(x)
 max_pool = GlobalMaxPooling1DMasked()(x)
 conc = concatenate([avg_pool, max_pool])
 outp = Dense(6, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model
model = get_model()
print("
Splitting the training dataset into actual training and validation datasets")
 X_train, X_val, y_train, y_val = train_test_split(train, target, test_size=0.2, random_state=42)
 assert(train.shape[0] == X_train.shape[0] + X_val.shape[0])
 assert(X_train.shape[1] == train.shape[1])
 assert(X_val.shape[1] == train.shape[1])
 assert(target.shape[0] == y_train.shape[0] + y_val.shape[0])
 print("training dataset has {0:,} samples and {1} features.".format(X_train.shape[0], X_train.shape[1]))
print("validating dataset has {0:,} samples and {1} features.".format(X_val.shape[0], X_val.shape[1]))<define_variables>
batch_size = 32
epochs = 4
file_path="weights_base.best.hdf5"
checkpoint = ModelCheckpoint(file_path, monitor='val_loss', verbose=1, save_best_only=True, mode='min')
 early = EarlyStopping(monitor="val_loss", mode="min", patience=20)
 callbacks_list = [checkpoint, early]
X_tra, X_val, y_tra, y_val = train_test_split(X_train, y_train, train_size=0.95)
 RocAuc = RocAucCallback(validation_data=(X_val, y_val), interval=1)
 hist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),
callbacks=callbacks_list)
 model.load_weights(file_path )
<train_on_grid><EOS> 
y_pred = model.predict(X_test, batch_size=2048)
 submission[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]] = y_pred
submission.to_csv('submission.csv', index=False )