Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<load_from_csv>
np.random.seed(42)
 warnings.filterwarnings('ignore')
 os.environ['OMP_NUM_THREADS'] = '4'
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<categorify>
EMBEDDING_FILE = '.. /input/fatsttext-common-crawl/crawl-300d-2M/crawl-300d-2M.vec'
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(EMBEDDING_FILE))
for f in object_list:
le = LabelEncoder()
le.fit(list(train[f].values)+ list(test[f].values))
train[f] = le.transform(list(train[f].values))
test[f] = le.transform(list(test[f].values))
Y = train['y']
train.drop(columns=['y'] , inplace = True , axis=1)
 combine = pd.concat([train , test])
 for f in object_list:
temp = pd.get_dummies(combine[f])
 combine = pd.concat([combine,temp] , axis =1)
 train=combine[:train.shape[0]]
test=combine[train.shape[0]:]<remove_duplicates>
train = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 test = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv')
 submission = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv' )
def df_column_uniquify(df):
df_columns = df.columns
new_columns = []
for item in df_columns:
counter = 0
newitem = item
while newitem in new_columns:
counter += 1
newitem = "{}_{}".format(item, counter)
 new_columns.append(newitem)
 df.columns = new_columns
return df
train = df_column_uniquify(train)
 test = df_column_uniquify(test)
 train['y']=Y<drop_column>
X_train = train["comment_text"].fillna("fillna" ).values
y_train = train[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values
X_test = test["comment_text"].fillna("fillna" ).values
original_col = list(test.drop(columns=object_list ).columns)
 display(train.head())
 display(test.head() )<categorify>
max_features = 30000
maxlen = 120
embed_size = 300
def get_additional_features(train , test , ID = False):
col = list(test.columns)
 if ID!= True:
col.remove('ID')
 n_comp = 12
tsvd = TruncatedSVD(n_components = n_comp , random_state = 98)
 tsvd_result_train = tsvd.fit_transform(train[col])
 tsvd_result_test = tsvd.transform(test[col])
 pca = PCA(n_components = n_comp , random_state = 98)
 pca_result_train = pca.fit_transform(train[col])
 pca_result_test = pca.transform(test[col])
 ica = FastICA(n_components =n_comp , random_state = 98)
 ica_result_train = ica.fit_transform(train[col])
 ica_result_test = ica.transform(test[col])
 grp = GaussianRandomProjection(n_components = n_comp , random_state = 98)
 grp_result_train = grp.fit_transform(train[col])
 grp_result_test = grp.transform(test[col])
 srp = SparseRandomProjection(n_components = n_comp , random_state = 98 , dense_output =True)
 srp_result_train = srp.fit_transform(train[col])
 srp_result_test = srp.transform(test[col])
 for i in range(1,n_comp+1):
train['tsvd_' + str(i)] = tsvd_result_train[:, i - 1]
test['tsvd_' + str(i)] = tsvd_result_test[:, i - 1]
train['pca_' + str(i)] = pca_result_train[:, i - 1]
test['pca_' + str(i)] = pca_result_test[:, i - 1]
train['ica_' + str(i)] = ica_result_train[:, i - 1]
test['ica_' + str(i)] = ica_result_test[:, i - 1]
train['grp_' + str(i)] = grp_result_train[:, i - 1]
test['grp_' + str(i)] = grp_result_test[:, i - 1]
train['srp_' + str(i)] = srp_result_train[:, i - 1]
test['srp_' + str(i)] = srp_result_test[:, i - 1]
return train ,test 
<train_model>
tokenizer = text.Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(X_train)+ list(X_test))
X_train = tokenizer.texts_to_sequences(X_train)
 X_test = tokenizer.texts_to_sequences(X_test)
 x_train = sequence.pad_sequences(X_train, maxlen=maxlen)
 x_test = sequence.pad_sequences(X_test, maxlen=maxlen)
def get_lgb_data(train , test , col , label , params ,rounds):
i=0
RMSE = []
R2_Score = []
ID = []
kf = KFold(n_splits = 5 , shuffle = False)
 train = train.reset_index(drop = True)
 for train_index , test_index in kf.split(train):
train_x , test_x = train.iloc[train_index, :] ,test.iloc[test_index ,:]
train_y , test_y = label.iloc[train_index] , label.iloc[test_index]
train_lgb = lgb.Dataset(train_x[col] , train_y)
 model = lgb.train(params , train_lgb , num_boost_round = rounds)
 pred = model.predict(test_x[col])
 test_x['label'] = list(test_y)
 test_x['predicted'] = pred
r2 = r2_score(test_y , pred)
 rmse = MSE(test_y ,pred)**0.5
print('R2 Scored of Fold '+str(i+1)+' is '+str(r2))
R2_Score.append(r2)
 RMSE.append(rmse)
 print('RMSE of Fold '+str(i+1)+' is '+str(rmse))
ID.append(test_x['ID'])
 if i==0:
Final = test_x
else:
Final = Final.append(test_x,ignore_index=True)
 i+=1
lgb_train = lgb.Dataset(train[col], label)
 model = lgb.train(params , lgb_train , num_boost_round = rounds)
 pred = model.predict(test[col])
 lgb.plot_importance(model, max_num_features = 20)
 Final_pred = pd.DataFrame({'ID': test['ID'] ,'y':pred})
 print('Out of Bag R2 Score')
 print(np.mean(r2))
print('Out of Bag RMSE')
 print(np.mean(RMSE))
return Final , Final_pred<predict_on_test>
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector.copy()
def get_sklearn_data(train , test , model , label ,col):
ID = []
RMSE = []
R2_Score = []
i=0
train =train.reset_index(drop = True)
 kf = KFold(n_splits = 5 , shuffle = False)
 for train_index , test_index in kf.split(train):
train_x , test_x = train.iloc[train_index,:] , train.iloc[test_index,:]
train_y , test_y = label.iloc[train_index] , label.iloc[test_index]
model.fit(train_x[col],train_y)
 pred = model.predict(test_x[col])
 test_x['label'] = list(test_y)
 test_x['predicted'] = pred
r2 = r2_score(test_y , pred)
 rmse = MSE(test_y , pred)**0.5
print('R2 Scored of Fold '+str(i+1)+' is '+str(r2))
R2_Score.append(r2)
 RMSE.append(rmse)
 print('RMSE of Fold '+str(i+1)+' is '+str(rmse))
ID.append(test_x['ID'])
 if i==0:
Final = test_x
else:
Final = Final.append(test_x,ignore_index=True)
 i+=1
model.fit(train[col] , label)
 Final_pred = model.predict(test[col])
 Final_pred = pd.DataFrame({'ID':test['ID'] , 'y':Final_pred})
 print('In of Bag R2 Score')
 print(r2_score(label , model.predict(train[col])))
 print('Out of Bag R2 Score')
 print(np.mean(R2_Score))
print('In of Bag MSE Score')
 print(MSE(label , model.predict(train[col])))
 print('Out of Bag MSE Score')
 print(np.mean(RMSE))
return Final , Final_pred , model<merge>
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: %d - score: %.6f
" %(epoch+1, score))
def magic_function(train ,test , columns):
for col in columns:
magic = train[['ID' , col , 'y']]
magic = magic.groupby([col])['y'].mean()
col_name = str(col)+'_mean'
magic = pd.DataFrame({col: magic.index , col_name:list(magic)})
 magic_mean = magic[col_name].mean()
train = train.merge(magic , how = 'left' , on = col)
 test = test.merge(magic , how = 'left' , on = col)
 test[col_name] = test[col_name].fillna(magic_mean)
 return train , test<feature_engineering>
def get_model() :
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(GRU(80, return_sequences=True))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 outp = Dense(6, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model
model = get_model()
train, test = magic_function(train , test , object_list)
 train.sample(5)
 train_new , test_new = get_additional_features(train.drop(columns = object_list),test.drop(columns = object_list))
train_new = train_new.sample(frac = 1 , random_state = 98)
 train_new.sample(10)
<drop_column>
batch_size = 32
epochs = 2
X_tra, X_val, y_tra, y_val = train_test_split(x_train, y_train, train_size=0.85, random_state=233)
 RocAuc = RocAucEvaluation(validation_data=(X_val, y_val), interval=1 )
train_new.drop(columns=columns_drop , inplace = True)
 test_new.drop(columns = columns_drop , inplace = True)
 col = list(test_new.columns )<load_pretrained>
hist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=1, validation_data=(X_val, y_val),
callbacks=[RocAuc], verbose=2 )
%%time
lasso1 = Lasso(alpha = 5 , random_state = 98)
 lasso_train , lasso_test , model = get_sklearn_data(train_new , test_new , lasso1 , train_new['y'] , col)
<load_pretrained>
y_pred = model.predict(x_test, batch_size=1024,verbose=99)
 submission[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]] = y_pred
submission.to_csv('submission.csv', index=False )