Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<set_options>
print(os.listdir(".. /input"))
rcParams['figure.figsize'] = 15, 7
res = stats.probplot(df['y'], plot=plt )<define_variables>
EMBEDDING_FILE = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
train = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 test = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv' )
temp = []
for i in df_num.columns:
if df[i].var() ==0:
temp.append(i)
 print(len(temp))
print(temp )<drop_column>
train["comment_text"].fillna("fillna")
 test["comment_text"].fillna("fillna")
 X_train = train["comment_text"].str.lower()
y_train = train[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values
X_test = test["comment_text"].str.lower()
count=0
low_var_col=[]
for i in test.columns:
if test[i].dtype == 'int64':
if test[i].var() <0.01:
low_var_col.append(i)
 count+=1
print(count)
 df.drop(low_var_col,axis=1,inplace=True)
 df_num.drop(low_var_col,axis=1,inplace=True)
 test.drop(low_var_col,axis=1,inplace=True )<feature_engineering>
max_features=100000
maxlen=150
embed_size=300
dic={}
for i in df_num.columns:
if i!='y':
if df[i].corr(df.y)>0.25 or df[i].corr(df.y)<-0.25:
dic[i]=df[i].corr(df.y)
 print("Important Features with there respective correlations are ",'
','---------------------------------------------------------','
',dic )<define_variables>
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: {:d} - score: {:.6f}".format(epoch+1, score))
d = {}; done = []
cols = df.columns.values
for c in cols: d[c]=[]
for i in range(len(cols)) :
if i not in done:
for j in range(i+1, len(cols)) :
if all(df[cols[i]] == df[cols[j]]):
done.append(j)
 d[cols[i]].append(cols[j])
 dub_cols = []
for k in d.keys() :
if len(d[k])> 0:
dub_cols += d[k]
print('Dublicates:','
', dub_cols )<drop_column>
tok=text.Tokenizer(num_words=max_features,lower=True)
 tok.fit_on_texts(list(X_train)+list(X_test))
X_train=tok.texts_to_sequences(X_train)
 X_test=tok.texts_to_sequences(X_test)
 x_train=sequence.pad_sequences(X_train,maxlen=maxlen)
 x_test=sequence.pad_sequences(X_test,maxlen=maxlen )
df.drop(['X279','X76','X37','X134','X147','X222','X244','X326'] , axis=1 , inplace=True )<drop_column>
embeddings_index = {}
with open(EMBEDDING_FILE,encoding='utf8')as f:
for line in f:
values = line.rstrip().rsplit(' ')
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
test.drop(['X279','X76','X37','X134','X147','X222','X244','X326'] , axis=1 , inplace=True)
 df_num.drop(['X279','X76','X37','X134','X147','X222','X244','X326'] , axis=1 , inplace=True )<categorify>
word_index = tok.word_index
num_words = min(max_features, len(word_index)+ 1)
 embedding_matrix = np.zeros(( num_words, embed_size))
for word, i in word_index.items() :
if i >= max_features:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
categorical=[]
for i in df.columns:
if df[i].dtype=='object':
le = preprocessing.LabelEncoder()
le.fit(list(df[i].values)+ list(test[i].values))
print("Categories in the encoded order from 1 to the size of "+i+" are : ")
 print(le.classes_)
 print("--------------------------------------------------------------------------")
 df[i] = le.transform(list(df[i].values))
test[i] = le.transform(list(test[i].values))
categorical.append(i )<train_model>
sequence_input = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix],trainable = False )(sequence_input)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(GRU(128, return_sequences=True,dropout=0.1,recurrent_dropout=0.1))(x)
 x = Conv1D(64, kernel_size = 3, padding = "valid", kernel_initializer = "glorot_uniform" )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 x = concatenate([avg_pool, max_pool])
 preds = Dense(6, activation="sigmoid" )(x)
 model = Model(sequence_input, preds)
 model.compile(loss='binary_crossentropy',optimizer=Adam(lr=1e-3),metrics=['accuracy'] )
train_y = df["y"].values
train_X = df.drop(['y'], axis=1)
 def xgb_r2_score(preds, final):
labels = dtrain.get_label()
return 'r2', r2_score(labels, preds)
 xgb_params = {
'n_trees': 520,
'eta': 0.0045,
'max_depth': 4,
'subsample': 0.98,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'base_score': np.mean(train_y),
'silent': 1
}
final = xgb.DMatrix(train_X, train_y, feature_names=train_X.columns.values)
 model = xgb.train(dict(xgb_params), final, num_boost_round=200, feval=xgb_r2_score, maximize=True)
 fig, ax = plt.subplots(figsize=(10,10))
xgb.plot_importance(model, max_num_features=40, height=0.8, ax=ax, color = 'coral')
 print("Feature Importance by XGBoost")
 plt.show()
model = RandomForestRegressor(n_estimators=200, max_depth=10, min_samples_leaf=4, max_features=0.2, n_jobs=-1, random_state=0)
 model.fit(train_X, train_y)
 feat_names = train_X.columns.values
importances = model.feature_importances_
std = np.std([tree.feature_importances_ for tree in model.estimators_], axis=0)
 indices = np.argsort(importances)[::-1][:40]
plt.subplots(figsize=(10,10))
plt.title("Feature importances by RandomForestRegressor")
 plt.ylabel("Features")
 plt.barh(range(len(indices)) , importances[indices], color="green", align="center")
 plt.yticks(range(len(indices)) , feat_names[indices], rotation='horizontal')
 plt.ylim([-1, len(indices)])
 plt.show()<feature_engineering>
batch_size = 128
epochs = 4
X_tra, X_val, y_tra, y_val = train_test_split(x_train, y_train, train_size=0.9, random_state=233 )
df['X314_plus_X315'] = df.apply(lambda row: row.X314 + row.X315, axis=1)
 test['X314_plus_X315'] = test.apply(lambda row: row.X314 + row.X315, axis=1 )<compute_test_metric>
model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),callbacks = callbacks_list,verbose=1)
 model.load_weights(filepath)
 print('Predicting.... ')
 y_pred = model.predict(x_test,batch_size=1024,verbose=1 )
<feature_engineering><EOS> 
submission = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv')
 submission[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]] = y_pred
submission.to_csv('submission.csv', index=False )