Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<import_modules>
warnings.filterwarnings('ignore')
 warnings.filterwarnings('ignore' )
from torch.utils.data import TensorDataset, DataLoader
from torch.autograd.variable import Variable<create_dataframe>
train = pd.read_csv(".. /input/jigsawtraintest/train_jigsaw.csv")
 test= pd.read_csv(".. /input/jigsawtraintest/test_jigsaw.csv")
 pos_df = pd.read_csv(".. /input/pos-neg-files/positive words.txt", sep="
", header=None)
 neg_df = pd.read_csv(".. /input/pos-neg-files/Negative words.txt", sep="
", header=None, encoding = "ISO-8859-1")
 pos_df.columns = ['words']
neg_df.columns = ['words']
Ttrain = TensorDataset(torch.DoubleTensor(np.array(xtrain.values, dtype="float32")) ,
torch.LongTensor(np.array(ytrain.values, dtype="float32")-1))<load_pretrained>
for df in [train, test]:
print(df.isna().sum() )
trainLoader = DataLoader(Ttrain, batch_size = 20, shuffle=True )<choose_model_class>
labels = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
class Net(nn.Module):
def __init__(self, n_cols):
super(Net, self ).__init__()
self.first = nn.Sequential(
 nn.BatchNorm1d(n_cols),
nn.Linear(n_cols, 10),
nn.ReLU() ,
nn.Dropout(p=0.5),
nn.Linear(10, 4),
nn.Softmax(dim=1)
)
 def forward(self, x):
return self.first(x)
 net = Net(len(xtrain.columns)).double()<choose_model_class>
for col in labels:
print(train[col].value_counts())
 print("
" )
loss = nn.CrossEntropyLoss()
metrics = [F1score]
opt = optim.Adam(net.parameters() , weight_decay=1e-3)
<train_model>
stop_words = set(stopwords.words('english'))
for df in [train, test]:
df['words_length'] = df['comment_text'].apply(lambda x: len(x))
def fit(model, lr, xtr, ytr, xvl, yvl, train_dl, n_epochs, loss, opt, metrics, annln=False, mult_dec=True):

 if(annln): annl = lrAnnealing(lr, 40, 449, mult_dec)
 for epoch in range(n_epochs):
tl = iter(train_dl)
 length = len(train_dl)
 for t in range(length):
xt, yt = next(tl)
 y_pred = model(Variable(xt))
l = loss(y_pred, Variable(yt))
if(annln): annl(opt)
 opt.zero_grad()
l.backward()
opt.step()
val_score = get_f1score(model,
torch.DoubleTensor(np.array(xvl, dtype = "float32")) ,
torch.LongTensor(np.array(yvl, dtype = "float32")-1))
trn_score = get_f1score(model,
torch.DoubleTensor(np.array(xtr, dtype = "float32")) ,
torch.LongTensor(np.array(ytr, dtype = "float32")-1))
if(epoch+1)%5 == 0:
print("Epoch " + str(epoch)+ "::"
+ " trnF1score: " + str(trn_score)
 +", valF1score: " + str(val_score))
def get_f1score(model, x, y):

 pred = model(Variable(x ).contiguous())
 ypreds = np.argmax(pred.contiguous().data.numpy() , axis=1)
 yactuals = y.contiguous().numpy()
return F1score(yactuals, ypreds)
 def set_lr(opt, lr):

 for pg in opt.param_groups: pg['lr'] = lr
class lrAnnealing() :
def __init__(self, ini_lr, epochs, itr_per_epoch, mult_dec):

 self.epochs = epochs
self.ipe = itr_per_epoch
self.m_dec = mult_dec
self.ppw =(self.ipe * self.epochs)// 4
self.count = 0
self.lr = ini_lr
self.values = np.cos(np.linspace(np.arccos(self.lr), np.arccos(1e-10), self.ppw))
self.mult = 1
def __call__(self, opt):

 self.count += 1
set_lr(opt, self.values[self.count-1]*self.mult)
 if self.count == len(self.values):
self.count = 0
if(self.m_dec): self.mult /= 2<set_options>
for col in labels:
print(train.groupby([col])['words_length'].mean())
 print("
" )
warnings.filterwarnings("ignore", category=UndefinedMetricWarning )<train_model>
contractions = {
"ain't": "am not ",
"aren't": "are not",
"can't": "cannot",
"can't've": "cannot have",
"'cause": "because",
"could've": "could have",
"couldn't": "could not",
"couldn't've": "could not have",
"didn't": "did not",
"doesn't": "does not",
"don't": "do not",
"hadn't": "had not",
"hadn't've": "had not have",
"hasn't": "has not",
"haven't": "have not",
"he'd": "he would",
"he'd've": "he would have",
"he'll": "he will",
"he'll've": "he shall have",
"he's": "he is",
"how'd": "how did",
"how'd'y": "how do you",
"how'll": "how will",
"how's": "how is",
"i'd": "I would",
"i'd've": "I would have",
"i'll": "I will",
"i'll've": "I shall have",
"i'm": "I am",
"i've": "I have",
"isn't": "is not",
"it'd": "it would",
"it'd've": "it would have",
"it'll": "it will",
"it'll've": "it will have",
"it's": "it is",
"let's": "let us",
"ma'am": "madam",
"mayn't": "may not",
"might've": "might have",
"mightn't": "might not",
"mightn't've": "might not have",
"must've": "must have",
"mustn't": "must not",
"mustn't've": "must not have",
"needn't": "need not",
"needn't've": "need not have",
"o'clock": "of the clock",
"oughtn't": "ought not",
"oughtn't've": "ought not have",
"shan't": "shall not",
"sha'n't": "shall not",
"shan't've": "shall not have",
"she'd": "she would",
"she'd've": "she would have",
"she'll": "she will",
"she'll've": "she will have",
"she's": "she is",
"should've": "should have",
"shouldn't": "should not",
"shouldn't've": "should not have",
"so've": "so have",
"so's": "so is",
"that'd": "that would",
"that'd've": "that would have",
"that's": "that is",
"there'd": "there would",
"there'd've": "there would have",
"there's": "there is",
"they'd": "they would",
"they'd've": "they would have",
"they'll": "they will",
"they'll've": "they will have",
"they're": "they are",
"they've": "they have",
"to've": "to have",
"wasn't": "was not",
"we'd": "we would",
"we'd've": "we would have",
"we'll": "we will",
"we'll've": "we will have",
"we're": "we are",
"we've": "we have",
"weren't": "were not",
"what'll": "what will",
"what'll've": "what will have",
"what're": "what are",
"what's": "what is",
"what've": "what have",
"when's": "when is",
"when've": "when have",
"where'd": "where did",
"where's": "where is",
"where've": "where have",
"who'll": "who will",
"who'll've": "who will have",
"who's": "who is",
"who've": "who have",
"why's": "why is",
"why've": "why have",
"will've": "will have",
"won't": "will not",
"won't've": "will not have",
"would've": "would have",
"wouldn't": "would not",
"wouldn't've": "would not have",
"y'all": "you all",
"y'all'd": "you all would",
"y'all'd've": "you all would have",
"y'all're": "you all are",
"y'all've": "you all have",
"you'd": "you would",
"you'd've": "you would have",
"you'll": "you will",
"you'll've": "you will have",
"you're": "you are",
"you've": "you have"
}
%time fit(net, 1e-2, xtrain, ytrain, xvalid, yvalid, trainLoader, 40, loss, opt, metrics, annln=False )<train_model>
def preprocess_text_for_word_embeddings(text):
for word in text.split() :
if word.lower() in contractions:
text = text.replace(word, contractions[word.lower() ])
 text = re.sub(r'\d+', '', text)
 text = re.sub('[^A-Za-z0-9]+', ' ', text)
 text = text.strip()
return text
%time fit(net, 1e-3, xtrain, ytrain, xvalid, yvalid, trainLoader, 40, loss, opt, metrics, annln=False )<train_model>
full_text = [i for i in train['comment_text']] + [i for i in test['comment_text']]
full_text = [(lambda x: preprocess_text_for_word_embeddings(x))(x)for x in full_text]
%time fit(net, 1e-4, xtrain, ytrain, xvalid, yvalid, trainLoader, 40, loss, opt, metrics, annln=False )<train_model>
train['clean_text'] = full_text[:len(train)]
test['clean_text'] = full_text[len(train):]
%time fit(net, 1e-5, xtrain, ytrain, xvalid, yvalid, trainLoader, 40, loss, opt, metrics, annln=False )<train_model>
def tfidf_transform(clean_text):
vectorizer = TfidfVectorizer(stop_words = set(stopwords.words('english')))
 vectorizer.fit(clean_text)
 x_train_tfidf_vec = vectorizer.transform(clean_text[:len(train)])
 x_test_tfidf_vec = vectorizer.transform(clean_text[len(train):])
 print(x_train_tfidf_vec.shape, x_test_tfidf_vec.shape)
 return x_train_tfidf_vec, x_test_tfidf_vec
%time fit(net, 1e-6, xtrain, ytrain, xvalid, yvalid, trainLoader, 40, loss, opt, metrics, annln=False )<train_model>
x_train_tfidf_vec, x_test_tfidf_vec = tfidf_transform(lower_case_text )
<train_model>
clf = LogisticRegression()
grid_values = {'penalty': ['l1', 'l2'],'C':[0.001,.009,0.01,.09,1,5,10,25]}
grid_clf = GridSearchCV(clf, param_grid = grid_values,scoring = 'recall')
 grid_clf.fit(x_train_tfidf_vec, train['toxic'] )
<train_model>
y_pred_acc = grid_clf.predict_proba(x_test_tfidf_vec)[:,1]
roc_auc_score(test['toxic'], y_pred_acc )
<train_model>
y_prediction = pd.DataFrame(columns=labels)
 for col in labels:
model = LogisticRegression()
model.fit(x_train_tfidf_vec, train[col])
 y_prediction[col] = model.predict_proba(x_test_tfidf_vec)[:,1]
print(col, " ROC AUC Score = ",roc_auc_score(test[col], y_prediction[col]))
<save_to_csv>
svd = TruncatedSVD(n_components = 100)
 x_train_tfidf_vec = svd.fit_transform(x_train_tfidf_vec)
 x_test_tfidf_vec = svd.fit_transform(x_test_tfidf_vec )
m = RandomForestClassifier(n_estimators=100, min_samples_leaf=25, max_features=0.5, n_jobs=-1)
 m.fit(xtrain, ytrain)
 print_score(m,(xtrain, ytrain),(xvalid, yvalid))
to_pred = test[xtrain.columns]
npArray = np.stack([lb3.inverse_transform(test['Id'].values), m.predict(to_pred)], axis=-1)
 res = pd.DataFrame(npArray, index=np.arange(len(npArray)) , columns=['Id', 'Target'])
 res.to_csv("submission1.csv", index=False )<save_to_csv>
m2 = RandomForestClassifier(n_estimators=100, min_samples_leaf=25, max_features=0.5, n_jobs=-1)
 m2.fit(xtrain2, ytrain2)
 to_pred = test[xtrain2.columns]
npArray = np.stack([lb3.inverse_transform(test['Id'].values), m2.predict(to_pred)], axis=-1)
 res = pd.DataFrame(npArray, index=np.arange(len(npArray)) , columns=['Id', 'Target'])
 res.to_csv("submission2.csv", index=False )<save_to_csv>
x_train_vec, x_val_vec, y_train, y_val = train_test_split(x_train_tfidf_vec, train[labels],
test_size=0.2,
random_state=0 )
m3 = RandomForestClassifier(n_estimators=100, min_samples_leaf=25, max_features=0.5, n_jobs=-1, class_weight='balanced')
 m3.fit(xtrain3, ytrain3)
 to_pred = test[xtrain3.columns]
npArray = np.stack([lb3.inverse_transform(test['Id'].values), m3.predict(to_pred)], axis=-1)
 res = pd.DataFrame(npArray, index=np.arange(len(npArray)) , columns=['Id', 'Target'])
 res.to_csv("submission3.csv", index=False )<save_to_csv>
x_train_text, x_val_text, _, _ = train_test_split(train['clean_text'], train[labels],
test_size=0.2,
random_state=0 )
pred = net(Variable(torch.DoubleTensor(np.array(test.drop(['Id', 'idhogar'], axis=1 ).values, dtype="float32")) ).contiguous())
 ypreds = pred.contiguous().data.numpy().argmax(1)+ 1
npArray = np.stack([lb3.inverse_transform(test['Id'].values),ypreds], axis=-1); npArray[0]
res = pd.DataFrame(npArray, index=np.arange(len(npArray)) , columns=['Id', 'Target'])
 res.to_csv("submission4.csv", index=False )<save_to_csv>
cv1 = CountVectorizer()
cv1.fit(x_train_text)
 cv2 = CountVectorizer()
cv2.fit(x_val_text)
 cv3 = CountVectorizer()
cv3.fit(test['clean_text'])
 print("Train Set Vocabulary Size:", len(cv1.vocabulary_))
print("Val Set Vocabulary Size:", len(cv2.vocabulary_))
print("Test Set Vocabulary Size:", len(cv3.vocabulary_))
print("Number of Words that occur in both:",
len(set(cv1.vocabulary_.keys() ).intersection(set(cv3.vocabulary_.keys()))))
ypreds = classifier.predict(test[xtrain.columns])+ 1
npArray = np.stack([lb3.inverse_transform(test['Id'].values),ypreds], axis=-1)
 res = pd.DataFrame(npArray, index=np.arange(len(npArray)) , columns=['Id', 'Target'])
 res.to_csv("submission5.csv", index=False )<save_to_csv>
EMBEDDING_FILE = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
EMBEDDING_DIM = 300
all_words = set(cv1.vocabulary_.keys() ).union(set(cv2.vocabulary_.keys())).union(set(cv3.vocabulary_.keys()))
ypreds = classifier2.predict(test[xtrain2.columns])+ 1
npArray = np.stack([lb3.inverse_transform(test['Id'].values),ypreds], axis=-1)
 res = pd.DataFrame(npArray, index=np.arange(len(npArray)) , columns=['Id', 'Target'])
 res.to_csv("submission6.csv", index=False )<set_options>
def get_embedding() :
embeddings_index = {}
f = open(EMBEDDING_FILE)
 for line in f:
values = line.split()
word = values[0]
if len(values)== EMBEDDING_DIM + 1 and word in all_words:
coefs = np.asarray(values[1:], dtype="float32")
 embeddings_index[word] = coefs
f.close()
return embeddings_index
embeddings_index = get_embedding()
print("Number of words that don't exist in GLOVE:", len(all_words - set(embeddings_index)) )
style.use('seaborn')
 print(os.listdir(".. /input"))
%matplotlib inline
pd.options.display.float_format = '{:,.2f}'.format
random_state = 5<define_variables>
MAX_SEQUENCE_LENGTH = 200
PATH = '.. /input/'<load_from_csv>
tokenizer = Tokenizer()
np_text = np.append(x_train_text.values,x_val_text.values)
 tokenizer.fit_on_texts(np.append(np_text, test['clean_text'].values))
word_index = tokenizer.word_index
nb_words = len(word_index)+ 1
embedding_matrix = np.random.rand(nb_words, EMBEDDING_DIM + 2)
 for word, i in word_index.items() :
embedding_vector = embeddings_index.get(word)
 sent = textblob.TextBlob(word ).sentiment
if embedding_vector is not None:
embedding_matrix[i] = np.append(embedding_vector, [sent.polarity, sent.subjectivity])
 else:
embedding_matrix[i, -2:] = [sent.polarity, sent.subjectivity]
train_seq = pad_sequences(tokenizer.texts_to_sequences(x_train_text), maxlen=MAX_SEQUENCE_LENGTH)
 val_seq = pad_sequences(tokenizer.texts_to_sequences(x_val_text), maxlen=MAX_SEQUENCE_LENGTH)
 test_seq = pad_sequences(tokenizer.texts_to_sequences(test['clean_text']), maxlen=MAX_SEQUENCE_LENGTH )
files = ['train', 'test', 'sample_submission']
tabs = [pd.read_csv(f'{PATH}{f}.csv', low_memory=False)for f in files]
for t in tabs: display(t.head() )<define_variables>
def build_model() :
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,))
x = Embedding(nb_words, EMBEDDING_DIM + 2, weights=[embedding_matrix],trainable = True )(sequence_input)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(GRU(128, return_sequences=True,dropout=0.1,recurrent_dropout=0.1))(x)
 x = Conv1D(64, kernel_size = 3, padding = "valid", kernel_initializer = "glorot_uniform" )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 x = concatenate([avg_pool, max_pool])
 dropout = SpatialDropout1D(0.2)
 dense_input = Input(shape=(100,))
dense_vector = BatchNormalization()(dense_input)
 feature_vector = concatenate([x, dense_vector])
 feature_vector = Dense(128, activation="relu" )(feature_vector)
 output = Dense(6, activation="sigmoid" )(feature_vector)
 model = Model(inputs=[sequence_input, dense_input], outputs=output)
 return model
data = [train, test]<count_missing_values>
model = build_model()
model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])
 early_stopping = EarlyStopping(monitor="val_acc", patience=2, verbose=1)
 print("Training the model")
 model.fit([train_seq, x_train_vec], y_train.values, validation_data=([val_seq, x_val_vec], y_val.values),epochs=2,
batch_size=1024, shuffle=True, callbacks=[early_stopping], verbose=1 )
for t in data: display(t.isna().sum() [t.isna().sum() != 0] / t.shape[0] )<groupby>
test_preds = model.predict([test_seq, x_test_tfidf_vec], batch_size=1024, verbose=1 )
for t in data: display(train[['age', 'rez_esc']].groupby('age' ).mean().head(25))<correct_missing_values>
labels = ['toxic','severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
for t in data: t['rez_esc'].fillna(0, inplace=True )<categorify>
y_df = np.where(test_preds > 0.5, 1, 0)
 y_df = pd.DataFrame(y_df, columns=['toxic','severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate'])
 y_df = y_df.astype('int' )
for t in data:
mask = t['v2a1'].isna()
name = "Training" if "Target" in t.columns else "Test"
print(f )<correct_missing_values>
def get_metri_scores(y_test, y_test_pred):
vals = precision_recall_fscore_support(y_test, y_test_pred, average='macro')
 precision = vals[0]
recall = vals[1]
f1 = vals[2]
acc = accuracy_score(y_test, y_test_pred)
 return precision, recall, f1, acc
<groupby><EOS> 
results_cv = pd.DataFrame({'labels': labels})
 results_cv['acc'] = 0
results_cv['f1'] = 0
results_cv['precision'] = 0
results_cv['recall'] = 0
for col in labels:
print(col)
 precision, recall, f1, acc = get_metri_scores(test[col], y_df[col])
 results_cv['acc'][results_cv['labels']==col] = acc
results_cv['f1'][results_cv['labels']==col] = f1
results_cv['precision'][results_cv['labels']==col] = precision
results_cv['recall'][results_cv['labels']==col] = recall